From 7530f4c690a1bd64c6f3b846d43b0522a25aa722 Mon Sep 17 00:00:00 2001 From: nmzpy <246990748+nmzpy@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:48:31 +0300 Subject: [PATCH 01/11] feat: add codec-aware HEVC export and CUDA compositor performance --- electron/electron-env.d.ts | 65 +- electron/ipc/export/native-video.test.ts | 1083 ++++++- electron/ipc/export/native-video.ts | 1426 ++++++++- .../nativeStaticLayoutRoutePlan.test.ts | 75 +- .../ipc/export/nativeStaticLayoutRoutePlan.ts | 140 +- electron/ipc/nativeVideoExport.test.ts | 391 ++- electron/ipc/nativeVideoExport.ts | 179 +- electron/ipc/register/export.ts | 123 +- electron/ipc/state.ts | 23 +- .../nvidia-cuda-compositor/CMakeLists.txt | 18 + .../cursorTelemetry.mjs | 191 ++ .../cursorTelemetry.test.mjs | 169 + .../overlayManifest.mjs | 106 + .../overlayManifest.test.mjs | 294 ++ .../run-mp4-pipeline.mjs | 195 +- .../nvidia-cuda-compositor/sourcePtsPlan.mjs | 14 + .../sourcePtsPlan.test.mjs | 44 + .../native/nvidia-cuda-compositor/src/main.cu | 2805 ++++++++++++++++- .../temporalAccumulate.test.mjs | 529 ++++ electron/preload.ts | 424 ++- scripts/benchmark-cuda4k.mjs | 188 ++ scripts/build-nvidia-cuda-compositor.mjs | 161 +- .../video-editor/ExportSettingsMenu.tsx | 411 ++- src/components/video-editor/VideoEditor.tsx | 181 +- .../video-editor/editorPreferences.test.ts | 87 + .../video-editor/editorPreferences.ts | 148 +- .../video-editor/exportStartSettings.test.ts | 34 + .../video-editor/exportStartSettings.ts | 15 + .../video-editor/exportStatusModel.test.ts | 61 +- .../video-editor/exportStatusModel.ts | 50 +- .../video-editor/mp4ExportRouting.test.ts | 203 +- .../video-editor/mp4ExportRouting.ts | 56 +- .../video-editor/mp4ExportSettings.test.ts | 46 +- .../video-editor/mp4ExportSettings.ts | 72 +- .../video-editor/projectPersistence.test.ts | 53 +- .../video-editor/projectPersistence.ts | 60 +- .../video-editor/smokeExportConfig.test.ts | 28 +- .../video-editor/smokeExportConfig.ts | 44 +- .../video-editor/useNvidiaCudaExportOptIn.ts | 18 + .../videoPlayback/zoomTransform.test.ts | 78 +- .../videoPlayback/zoomTransform.ts | 54 +- src/i18n/locales/de/settings.json | 43 + src/i18n/locales/en/settings.json | 43 + src/i18n/locales/es/settings.json | 43 + src/i18n/locales/fr/settings.json | 43 + src/i18n/locales/it/settings.json | 43 + src/i18n/locales/ko/settings.json | 43 + src/i18n/locales/nl/settings.json | 43 + src/i18n/locales/pt-BR/settings.json | 43 + src/i18n/locales/ru/settings.json | 43 + src/i18n/locales/zh-CN/settings.json | 43 + src/i18n/locales/zh-TW/settings.json | 43 + src/lib/exporter/exportBitrate.test.ts | 118 +- src/lib/exporter/exportBitrate.ts | 43 +- src/lib/exporter/exportTuning.test.ts | 59 + src/lib/exporter/exportTuning.ts | 223 +- src/lib/exporter/index.ts | 17 + src/lib/exporter/modernFrameRenderer.test.ts | 48 +- src/lib/exporter/modernFrameRenderer.ts | 169 +- .../modernVideoExporter.fallback.test.ts | 301 ++ ...rnVideoExporter.nativeStaticLayout.test.ts | 426 ++- ...rnVideoExporter.overlayPreparation.test.ts | 403 +++ src/lib/exporter/modernVideoExporter.ts | 1321 +++++++- src/lib/exporter/nativeFrameCapture.ts | 26 +- .../nativeStaticLayoutOverlays.test.ts | 97 + .../exporter/nativeStaticLayoutOverlays.ts | 110 + src/lib/exporter/types.ts | 34 + 67 files changed, 13584 insertions(+), 626 deletions(-) create mode 100644 electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs create mode 100644 electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs create mode 100644 electron/native/nvidia-cuda-compositor/overlayManifest.mjs create mode 100644 electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs create mode 100644 electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs create mode 100644 electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs create mode 100644 electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs create mode 100644 scripts/benchmark-cuda4k.mjs create mode 100644 src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts create mode 100644 src/lib/exporter/nativeStaticLayoutOverlays.test.ts create mode 100644 src/lib/exporter/nativeStaticLayoutOverlays.ts diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 8a4eedd63..e3824f2a7 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -136,6 +136,10 @@ interface RendererNativeStaticLayoutChunkMetric { outputBytes: number; fallbackReason?: string; windowsGpuSummary?: RendererWindowsGpuExportSummary; + nvidiaCudaSummary?: { + success?: boolean; + outputCodec?: "h264" | "hevc"; + }; } interface RendererNativeStaticLayoutMetrics extends RendererFfmpegAudioMuxMetrics { @@ -156,6 +160,8 @@ interface RendererNativeStaticLayoutProgress { elapsedMs?: number; averageFps?: number; instantFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; intervalMs?: number; intervalFrames?: number; intervalDecodeWallMs?: number; @@ -352,12 +358,27 @@ interface Window { nativeStaticLayoutExport: (options: { sessionId?: string; inputPath: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; width: number; height: number; frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; durationSec: number; + overlayLayers?: Array<{ + id: string; + order: number; + path: string; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + pixelFormat: "rgba"; + }>; contentWidth: number; contentHeight: number; offsetX: number; @@ -399,7 +420,20 @@ interface Window { anchorY: number; aspectRatio: number; }>; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; + temporalBlur?: { + sampleCount: number; + shutterFraction: number; + weightCurvePower: number; + } | null; timelineSegments?: Array<{ sourceStartMs: number; sourceEndMs: number; @@ -425,6 +459,14 @@ interface Window { }) => Promise<{ success: boolean; tempPath?: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; + route?: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; encoderName?: string; error?: string; metrics?: RendererNativeStaticLayoutMetrics; @@ -442,20 +484,35 @@ interface Window { bitrate: number; encodingMode: "fast" | "balanced" | "quality"; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; }) => Promise<{ success: boolean; sessionId?: string; encoderName?: string; error?: string; }>; + nativeVideoExportOpenFrameChannel: (sessionId: string) => Promise<{ + success: boolean; + error?: string; + fallbackAvailable?: boolean; + }>; + nativeVideoExportWriteFrameViaChannel: ( + sessionId: string, + frameData: Uint8Array, + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; + nativeVideoExportWriteFramesViaChannel: ( + sessionId: string, + frameDataList: Uint8Array[], + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportWriteFrame: ( sessionId: string, frameData: Uint8Array, - ) => Promise<{ success: boolean; error?: string }>; + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportWriteFrames: ( sessionId: string, frameDataList: Uint8Array[], - ) => Promise<{ success: boolean; error?: string }>; + ) => Promise<{ success: boolean; error?: string; fallbackAvailable?: boolean }>; nativeVideoExportFinish: ( sessionId: string, options?: { @@ -735,7 +792,7 @@ interface Window { deleteRecordingFile: (filePath: string) => Promise<{ success: boolean; error?: string }>; getLocalMediaUrl: ( filePath: string, - ) => Promise<{ success: true; url: string } | { success: false }>; + ) => Promise<{ success: true; url: string; pending?: boolean } | { success: false }>; saveProjectFile: ( projectData: unknown, suggestedName?: string, diff --git a/electron/ipc/export/native-video.test.ts b/electron/ipc/export/native-video.test.ts index ee31dcdd7..c64532da7 100644 --- a/electron/ipc/export/native-video.test.ts +++ b/electron/ipc/export/native-video.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; vi.mock("electron", () => ({ app: { @@ -50,13 +50,18 @@ import { app } from "electron"; import { buildExperimentalNvidiaCudaStaticLayoutArgs, buildExperimentalWindowsGpuStaticLayoutArgs, + buildNativeStaticLayoutOverlayManifest, buildNativeStaticLayoutSourceProxyArgs, buildNativeStaticLayoutTimelineSegments, buildNativeVideoAudioMuxArgs, canCopyAudioCodecIntoMp4, + exportNativeStaticLayoutVideo, + formatNativeStaticLayoutZoomTelemetryLines, getExperimentalNvidiaCudaExportSkipReason, getNativeExportCapabilities, getNativeGpuCompositorStallTimeoutMs, + getNativeStaticLayoutOverlayExpectedSidecarBytes, + getNativeStaticLayoutRawFrameFallbackReason, getNativeStaticLayoutSourceProxyBitrate, getNvidiaCudaAudioExportSkipReason, getNvidiaCudaAutoStallTimeoutMs, @@ -65,6 +70,7 @@ import { mapNvidiaCudaWrapperProgressPercentage, muxExportedVideoAudioBuffer, type NativeStaticLayoutExportOptions, + type NativeStaticLayoutOverlayLayer, normalizeNativeStaticLayoutBackground, parseFfmpegDurationSeconds, parseFfmpegFrameRate, @@ -75,10 +81,16 @@ import { parseWindowsGpuExportProgressLine, parseWindowsGpuExportSummary, resolveExperimentalNvidiaCudaExportScriptPath, + resolveNativeStaticLayoutFpsFields, + resolveNvidiaCudaCursorAssets, + resolveNvidiaCudaNativeFps, + resolveNvidiaCudaNativeSummaryMetrics, + resolveNvidiaCudaOverlaySidecarSummaryMetrics, shouldCreateNativeStaticLayoutSourceProxy, validateNativeStaticLayoutSourceProxyMetadata, validateNativeVideoStreamStats, validateNvidiaCudaExportSummary, + validateNvidiaCudaStageMetricInvariants, validateWindowsGpuExportSummary, } from "./native-video"; @@ -410,6 +422,38 @@ describe("getNativeExportCapabilities", () => { process.platform === "win32" ? true : null, ); }); + + it("keeps CUDA available when the GPU probe is inconclusive so the live helper decides", async () => { + const envName = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + const originalEnv = process.env[envName]; + const originalIsPackaged = electronAppMock.isPackaged; + electronAppMock.isPackaged = true; + electronAppMock.getGPUInfo.mockRejectedValue(new Error("GPU info unavailable")); + fsMocks.access.mockResolvedValue(undefined); + delete process.env[envName]; + + try { + const capabilities = await getNativeExportCapabilities(); + if (process.platform === "win32") { + expect(capabilities.nvidiaCuda.available).toBe(true); + expect(capabilities.nvidiaCuda.hasNvidiaGpu).toBeNull(); + expect(capabilities.nvidiaCuda.skipReason).toBeNull(); + } else { + expect(capabilities.nvidiaCuda.available).toBe(false); + expect(capabilities.nvidiaCuda.skipReason).toBe("not-windows"); + } + } finally { + if (originalEnv === undefined) { + delete process.env[envName]; + } else { + process.env[envName] = originalEnv; + } + electronAppMock.isPackaged = originalIsPackaged; + electronAppMock.getGPUInfo.mockReset(); + electronAppMock.getGPUInfo.mockResolvedValue({ gpuDevice: [] }); + resetFsAccessMock(); + } + }); }); describe("getExperimentalNvidiaCudaExportSkipReason", () => { @@ -601,6 +645,261 @@ describe("resolveExperimentalNvidiaCudaExportScriptPath", () => { }); }); +describe("buildNativeStaticLayoutOverlayManifest", () => { + it("serializes sorted overlay layers into the CUDA manifest contract", () => { + expect( + buildNativeStaticLayoutOverlayManifest([ + { + id: "captions", + order: 2, + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]), + ).toEqual({ + layers: [ + { + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + }, + { + id: "captions", + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameCount: 60, + }, + ], + }); + }); + + it("passes effectiveFrameCount through while preserving the logical frameCount", () => { + expect( + buildNativeStaticLayoutOverlayManifest([ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }, + { + id: "captions", + order: 2, + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]), + ).toEqual({ + layers: [ + { + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + effectiveFrameCount: 41, + }, + { + id: "captions", + path: "captions.rgba", + x: 0, + y: 900, + width: 1920, + height: 180, + frameCount: 60, + }, + ], + }); + }); + + it("omits effectiveFrameCount for fully dynamic layers", () => { + const manifest = buildNativeStaticLayoutOverlayManifest([ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }, + ]); + + expect(manifest.layers[0]).toEqual({ + id: "effects", + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameCount: 60, + }); + expect("effectiveFrameCount" in manifest.layers[0]).toBe(false); + }); +}); + +describe("getNativeStaticLayoutOverlayExpectedSidecarBytes", () => { + const frameByteSize = 1920 * 1080 * 4; + + it("sizes deduped sidecars from effectiveFrameCount, not the logical frameCount", () => { + expect( + getNativeStaticLayoutOverlayExpectedSidecarBytes({ + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }), + ).toBe(frameByteSize * 41); + }); + + it("falls back to the logical frameCount for fully dynamic layers", () => { + expect( + getNativeStaticLayoutOverlayExpectedSidecarBytes({ + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 2, + frameCount: 60, + pixelFormat: "rgba", + }), + ).toBe(frameByteSize * 60); + }); +}); + +describe("native static-layout overlay sidecar validation", () => { + const frameByteSize = 1920 * 1080 * 4; + + function createOverlayLayer( + overrides: Partial = {}, + ): NativeStaticLayoutOverlayLayer { + return { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + ...overrides, + }; + } + + afterEach(() => { + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + }); + + it("rejects deduped sidecars truncated below effectiveFrameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 204 }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [ + createOverlayLayer({ frameCount: 300, effectiveFrameCount: 205 }), + ], + }), + ), + ).rejects.toThrow( + `Native overlay layer effects is truncated: expected ${frameByteSize * 205} bytes, received ${frameByteSize * 204}`, + ); + }); + + it("accepts deduped sidecars whose physical bytes match effectiveFrameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 205 }); + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createOverlayLayer({ frameCount: 300, effectiveFrameCount: 205 })], + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch(/truncated/i); + }); + + it("keeps validating fully dynamic layers against the logical frameCount", async () => { + fsMocks.stat.mockResolvedValue({ size: frameByteSize * 299 }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createOverlayLayer({ frameCount: 300 })], + }), + ), + ).rejects.toThrow( + `Native overlay layer effects is truncated: expected ${frameByteSize * 300} bytes, received ${frameByteSize * 299}`, + ); + }); +}); + describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { it("passes output canvas dimensions to the CUDA wrapper", () => { const args = buildExperimentalNvidiaCudaStaticLayoutArgs( @@ -612,6 +911,22 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { expect(args).toEqual(expect.arrayContaining(["--width", "1020", "--height", "572"])); }); + it("passes the high-level HEVC codec to the generalized CUDA compositor", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + "output.mp4", + "work", + ); + + expect(args).toEqual( + expect.arrayContaining(["--output-codec", "hevc", "--encoding-mode", "quality"]), + ); + expect(args).not.toContain("h264"); + }); + it("keeps explicit copy-source CUDA audio inline by default", () => { const args = buildExperimentalNvidiaCudaStaticLayoutArgs( createNvidiaCudaSkipOptions({ @@ -761,9 +1076,181 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { ]), ); }); + + it("passes the overlay sidecar manifest to the CUDA wrapper", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + overlayManifestPath: "overlay-manifest.json", + }), + "output.mp4", + "work", + ); + + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("passes the resolved temporal zoom blur plan to the CUDA wrapper", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + temporalBlur: { + sampleCount: 13, + shutterFraction: 0.62, + weightCurvePower: 1.5, + }, + }), + "output.mp4", + "work", + ); + + expect(args).toEqual( + expect.arrayContaining([ + "--temporal-blur-sample-count", + "13", + "--temporal-blur-shutter-fraction", + "0.62", + "--temporal-blur-weight-power", + "1.5", + ]), + ); + }); + + it("omits temporal blur args when no plan is configured", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({}), + "output.mp4", + "work", + ); + + expect(args).not.toContain("--temporal-blur-sample-count"); + }); + + it("keeps the D3D11 builder free of overlay manifest args", () => { + const args = buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + overlayManifestPath: "overlay-manifest.json", + }), + "output.mp4", + "work", + ); + + expect(args).not.toContain("--overlay-manifest"); + }); +}); + +describe("resolveNvidiaCudaCursorAssets", () => { + it("strips cursor asset paths when the CUDA route must not draw the cursor", () => { + const options = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + }); + + expect(resolveNvidiaCudaCursorAssets(options, true)).toEqual({ + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }); + }); + + it("preserves cursor asset paths when native cursor drawing is allowed", () => { + const options = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }); + + expect(resolveNvidiaCudaCursorAssets(options, false)).toEqual({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }); + }); + + it("normalizes missing cursor asset paths to null", () => { + const options = createNvidiaCudaSkipOptions({}); + expect(resolveNvidiaCudaCursorAssets(options, false)).toEqual({ + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }); + }); +}); + +describe("CUDA cursor telemetry contract", () => { + it("never emits --cursor-json for overlay exports after cursor assets are stripped", () => { + // Mirrors the reported failure: the Windows GPU prep leaves CSV telemetry + // paths on the options, overlay layers are present, and the CUDA wrapper + // must not receive the CSV file as --cursor-json (it JSON.parses the path). + const csvOptions = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + overlayManifestPath: "overlay-manifest.json", + }); + const cudaOptions = { + ...csvOptions, + ...resolveNvidiaCudaCursorAssets(csvOptions, true), + }; + + const args = buildExperimentalNvidiaCudaStaticLayoutArgs(cudaOptions, "output.mp4", "work"); + + expect(args).not.toContain("--cursor-json"); + expect(args).not.toContain("cursor-telemetry.csv"); + expect(args).not.toContain("--cursor-atlas-png"); + expect(args).not.toContain("--cursor-atlas-metadata"); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("emits --cursor-json only for non-overlay CUDA exports with prepared JSON telemetry", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorSize: 96, + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + }), + "output.mp4", + "work", + ); + + expect(args).toEqual(expect.arrayContaining(["--cursor-json", "cursor-telemetry.json"])); + expect(args).not.toContain("cursor-telemetry.csv"); + }); +}); + +describe("native static-layout encoder preference", () => { + it("requires rawvideo for CPU preference", () => { + expect( + getNativeStaticLayoutRawFrameFallbackReason({ + videoCodec: "hevc", + encoderPreference: "cpu", + }), + ).toBe("encoder-preference-cpu-requires-native-rawvideo"); + }); + + it("keeps HEVC hardware eligible for the CUDA route", () => { + expect( + getNativeStaticLayoutRawFrameFallbackReason({ + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ).toBeNull(); + }); }); describe("buildExperimentalWindowsGpuStaticLayoutArgs", () => { + it("rejects HEVC instead of constructing an H.264 GPU command", () => { + expect(() => + buildExperimentalWindowsGpuStaticLayoutArgs( + createNvidiaCudaSkipOptions({ videoCodec: "hevc" }), + "output.mp4", + ), + ).toThrow(/generalized NVIDIA CUDA compositor/i); + }); it("passes background blur to the D3D11 compositor", () => { const args = buildExperimentalWindowsGpuStaticLayoutArgs( createNvidiaCudaSkipOptions({ @@ -1101,11 +1588,190 @@ describe("parseNvidiaCudaExportSummary", () => { expect(summary?.nativeSummary?.fps).toBe(326.1); }); + it("passes through the extended additive compositor counters", () => { + const summary = parseNvidiaCudaExportSummary( + JSON.stringify({ + success: true, + fps: 30, + nativeSummary: { + success: true, + frames: 300, + totalMs: 920.5, + measuredFps: 326.1, + temporalBlurSampleCount: 13, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + overlayFileLoads: 4, + overlayCacheHits: 596, + compositeGpuMs: 240.25, + zoomBlurGpuMs: 12.5, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayBlendFrames: 300, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + compositeMs: 262.9, + flushMs: 2.4, + realtimeMultiplier: 32.6, + outputBytes: 1843200, + }, + }), + ); + + expect(summary?.nativeSummary).toMatchObject({ + totalMs: 920.5, + measuredFps: 326.1, + temporalBlurSampleCount: 13, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + overlayFileLoads: 4, + overlayCacheHits: 596, + compositeGpuMs: 240.25, + zoomBlurGpuMs: 12.5, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayBlendFrames: 300, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + compositeMs: 262.9, + flushMs: 2.4, + realtimeMultiplier: 32.6, + outputBytes: 1843200, + }); + }); + it("returns null when the wrapper output has no JSON object", () => { expect(parseNvidiaCudaExportSummary("native helper failed before summary")).toBeNull(); }); }); +describe("resolveNvidiaCudaNativeSummaryMetrics", () => { + it("maps only finite additive counters the helper actually reported", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + success: true, + totalMs: 920.5, + nvencMs: 512.4, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayFileLoads: 4, + overlayCacheHits: 596, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + // Legacy counters stay out of the metric mapping; the completion + // log surfaces them through explicit keys. + roiCompositeFrames: 300, + }), + ).toEqual({ + totalMs: 920.5, + nvencMs: 512.4, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + overlayFileLoads: 4, + overlayCacheHits: 596, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + }); + }); + + it("omits absent, non-finite, and non-numeric counters", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + temporalBlurSampleCount: Number.NaN, + temporalBlurFrames: Number.POSITIVE_INFINITY, + nvencMs: "512" as unknown as number, + overlayFileLoads: 0, + }), + ).toEqual({ overlayFileLoads: 0 }); + }); + + it("returns an empty object for missing native summaries", () => { + expect(resolveNvidiaCudaNativeSummaryMetrics(undefined)).toEqual({}); + }); +}); + +describe("resolveNvidiaCudaNativeFps", () => { + it("prefers the helper measured flush-span FPS over the encode-loop fps", () => { + expect( + resolveNvidiaCudaNativeFps({ + fps: 30, + nativeSummary: { fps: 320.5, measuredFps: 326.1 }, + }), + ).toBe(326.1); + }); + + it("falls back to the encode-loop fps and preserves the legacy nativeFps contract", () => { + expect(resolveNvidiaCudaNativeFps({ nativeSummary: { fps: 320.5 } })).toBe(320.5); + expect(resolveNvidiaCudaNativeFps({ nativeSummary: { fps: 0 } })).toBeUndefined(); + expect(resolveNvidiaCudaNativeFps(undefined)).toBeUndefined(); + }); +}); + +describe("validateNvidiaCudaStageMetricInvariants", () => { + it("accepts coherent additive stage metrics", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + totalMs: 920.5, + compositeGpuMs: 240.25, + overlayBlendGpuMs: 60.75, + overlayUploadMs: 18.1, + nvencMs: 512.4, + packetWriteMs: 8.2, + encodeMs: 700.1, + decodeWallMs: 80.3, + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 520, + temporalBlurBgPrecomposedFrames: 38, + }, + }), + ).toEqual([]); + }); + + it("rejects additive counters that exceed the helper wall time", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + totalMs: 100, + nvencMs: 512.4, + overlayUploadMs: 18.1, + }, + }), + ).toEqual(["nvencMs 512.4ms exceeds helper wall time 100ms"]); + }); + + it("rejects temporal blur counters that contradict the frame count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurSamplesTotal: 20, + }, + }), + ).toEqual(["temporalBlurSamplesTotal 20 below temporalBlurFrames 40"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 41, + }, + }), + ).toEqual(["temporalBlurBgPrecomposedFrames 41 exceeds temporalBlurFrames 40"]); + }); + + it("reports no issues when the native summary is absent", () => { + expect(validateNvidiaCudaStageMetricInvariants({})).toEqual([]); + }); +}); + describe("validateNvidiaCudaExportSummary", () => { it("accepts CUDA output when frames and stream durations match the export target", () => { const issues = validateNvidiaCudaExportSummary( @@ -1270,7 +1936,7 @@ describe("parseWindowsGpuExportProgressLine", () => { it("parses bounded helper progress lines", () => { expect( parseWindowsGpuExportProgressLine( - 'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0,"stage":"finalizing"}', + 'PROGRESS {"currentFrame":30,"totalFrames":60,"percentage":50,"averageFps":240.5,"instantFps":180.25,"intervalMs":166.4,"intervalFrames":30,"intervalEncodeMs":120.2,"intervalPipelineWaitMs":46.2,"intervalMonolithicCompositeFrames":0,"intervalZoomBlurFrames":13,"stage":"finalizing"}', ), ).toEqual({ currentFrame: 30, @@ -1284,6 +1950,7 @@ describe("parseWindowsGpuExportProgressLine", () => { intervalEncodeMs: 120.2, intervalPipelineWaitMs: 46.2, intervalMonolithicCompositeFrames: 0, + intervalZoomBlurFrames: 13, }); }); @@ -1309,6 +1976,102 @@ describe("parseWindowsGpuExportProgressLine", () => { ), ).toBeNull(); }); + describe("resolveNativeStaticLayoutFpsFields", () => { + it("prefers native measured FPS over the preparation-inclusive estimate", () => { + const progress = { + currentFrame: 300, + totalFrames: 600, + percentage: 50, + averageFps: 17.2, + instantFps: 19.1, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 45_000)).toEqual({ + averageFps: 17.2, + estimatedFps: undefined, + fpsSource: "native", + }); + }); + + it("never labels the preparation-inclusive estimate as measured encode speed", () => { + const progress = { + currentFrame: 120, + totalFrames: 600, + percentage: 20, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 30_000)).toEqual({ + averageFps: undefined, + estimatedFps: 4, + fpsSource: "estimated", + }); + }); + + it("reports no FPS fields before any frame has been encoded", () => { + expect( + resolveNativeStaticLayoutFpsFields( + { currentFrame: 0, totalFrames: 600, percentage: 0 }, + 12_000, + ), + ).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: undefined, + }); + }); + + it("keeps the estimate when only interval FPS is present but no helper average", () => { + const progress = { + currentFrame: 60, + totalFrames: 600, + percentage: 10, + instantFps: 240.5, + }; + + expect(resolveNativeStaticLayoutFpsFields(progress, 10_000)).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: "native", + }); + }); + }); + describe("formatNativeStaticLayoutZoomTelemetryLines", () => { + it("writes renderer zoom-blur columns for the CUDA compositor", () => { + const lines = formatNativeStaticLayoutZoomTelemetryLines([ + { + timeMs: 0, + scale: 1, + x: 0, + y: 0, + blurStrength: 0, + blurCenterX: 960, + blurCenterY: 540, + }, + { + timeMs: 33.333, + scale: 1.0123, + x: -11.8, + y: -6.6, + blurStrength: 0.00345, + blurCenterX: 960, + blurCenterY: 540, + }, + ]); + + expect(lines).toEqual([ + "0,1,0,0,0,960,540", + "33.333,1.0123,-11.8,-6.6,0.00345,960,540", + ]); + }); + + it("keeps 4-column telemetry backward compatible with blur defaults", () => { + const lines = formatNativeStaticLayoutZoomTelemetryLines([ + { timeMs: 0, scale: 1, x: 0, y: 0 }, + ]); + + expect(lines).toEqual(["0,1,0,0,0,0,0"]); + }); + }); }); describe("mapNvidiaCudaWrapperProgressPercentage", () => { @@ -1451,3 +2214,319 @@ describe("parseFfmpegFrameRate", () => { expect(parseFfmpegFrameRate("Video: h264")).toBeNull(); }); }); + +describe("native cursor atlas ownership", () => { + function createCursorOverlayLayer( + overrides: Partial = {}, + ): NativeStaticLayoutOverlayLayer { + return { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + ...overrides, + }; + } + + it("passes cursor assets and the overlay manifest through when the atlas owns the cursor", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.json", + cursorSize: 96, + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.tsv", + overlayManifestPath: "overlay-manifest.json", + cursorAtlasOwned: true, + }), + "output.mp4", + "work", + ); + + // The sidecar excluded cursor pixels, so the CUDA wrapper must receive both + // the overlay manifest and the cursor atlas: dropping either would lose the + // cursor and baking it again would double-render. + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + expect(args).toEqual( + expect.arrayContaining([ + "--cursor-json", + "cursor-telemetry.json", + "--cursor-atlas-png", + "cursor-atlas.png", + "--cursor-atlas-metadata", + "cursor-atlas.tsv", + ]), + ); + }); + + it("keeps stripping baked-cursor assets even when a manifest path is present", () => { + const csvOptions = createNvidiaCudaSkipOptions({ + cursorTelemetryPath: "cursor-telemetry.csv", + cursorAtlasPath: "cursor-atlas.png", + cursorAtlasMetadataPath: "cursor-atlas.csv", + overlayManifestPath: "overlay-manifest.json", + }); + const cudaOptions = { + ...csvOptions, + ...resolveNvidiaCudaCursorAssets(csvOptions, true), + }; + + const args = buildExperimentalNvidiaCudaStaticLayoutArgs(cudaOptions, "output.mp4", "work"); + + expect(args).not.toContain("--cursor-json"); + expect(args).not.toContain("--cursor-atlas-png"); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("refuses a native-owned cursor when the CUDA route cannot run", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createCursorOverlayLayer()], + cursorAtlasOwned: true, + experimentalWindowsGpuCompositor: false, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor/i, + ); + }); + + it("allows a native-owned cursor when the CUDA route is explicitly opted in on Windows", async () => { + if (process.platform !== "win32") { + return; + } + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + overlayLayers: [createCursorOverlayLayer()], + cursorAtlasOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + // The preflight guard must not reject the CUDA-opt-in route; any later + // failure is a runtime/skip error, not a cursor-ownership refusal. + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch( + /Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor/i, + ); + }); +}); + +describe("resolveNvidiaCudaOverlaySidecarSummaryMetrics", () => { + it("surfaces dimensions and physical/effective frame counts for deduped sidecars", () => { + expect( + resolveNvidiaCudaOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + effectiveFrameCount: 41, + pixelFormat: "rgba", + }, + ], + }), + ), + ).toEqual({ + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + }); + }); + + it("falls back to the logical frame count for fully dynamic sidecars", () => { + const metrics = resolveNvidiaCudaOverlaySidecarSummaryMetrics( + createNvidiaCudaSkipOptions({ + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + }, + ], + }), + ); + + expect(metrics).toEqual({ + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 300, + }); + expect("overlayEffectiveFrames" in metrics).toBe(false); + }); + + it("returns no metrics when overlay layers are absent", () => { + expect( + resolveNvidiaCudaOverlaySidecarSummaryMetrics(createNvidiaCudaSkipOptions({})), + ).toEqual({}); + }); +}); + +describe("native summary metric mapping (module 3 surface)", () => { + it("maps the additive temporal/overlay counters the helper emitted", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + temporalBlurStationaryFrames: 12, + temporalBgCacheBuilds: 14, + temporalBgCacheHits: 26, + overlayStaticRegionBlends: 598, + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + overlayBlendFrames: 300, + }), + ).toEqual({ + temporalBlurStationaryFrames: 12, + temporalBgCacheBuilds: 14, + temporalBgCacheHits: 26, + overlayStaticRegionBlends: 598, + overlayWidth: 1920, + overlayHeight: 1080, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + overlayBlendFrames: 300, + }); + }); + + it("keeps absent host-read/H2D fields out of the mapping until the helper emits them", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + overlayHostReadMs: Number.NaN, + overlayH2DEnqueueMs: 4.5, + }), + ).toEqual({ overlayH2DEnqueueMs: 4.5 }); + }); +}); + +describe("native summary metric invariants (module 3)", () => { + it("accepts coherent temporal cache and stationary counters", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurStationaryFrames: 10, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 8, + temporalBgCacheHits: 22, + overlayBlendFrames: 300, + overlayStaticRegionBlends: 297, + overlayFrameCount: 300, + overlayPhysicalFrames: 41, + overlayEffectiveFrames: 41, + }, + }), + ).toEqual([]); + }); + + it("rejects stationary frames beyond the temporal frame count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurStationaryFrames: 41, + }, + }), + ).toEqual(["temporalBlurStationaryFrames 41 exceeds temporalBlurFrames 40"]); + }); + + it("rejects temporal cache counters that exceed the precomposed frame budget", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 20, + temporalBgCacheHits: 15, + }, + }), + ).toEqual([ + "temporalBgCacheBuilds 20 + temporalBgCacheHits 15 exceed temporalBlurBgPrecomposedFrames 30", + ]); + }); + + it("rejects overlay static-region blends beyond overlay blend frames", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayBlendFrames: 300, + overlayStaticRegionBlends: 301, + }, + }), + ).toEqual(["overlayStaticRegionBlends 301 exceeds overlayBlendFrames 300"]); + }); + + it("rejects overlay sidecar frame counts that contradict the logical count", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayFrameCount: 300, + overlayPhysicalFrames: 301, + }, + }), + ).toEqual(["overlayPhysicalFrames 301 exceeds overlayFrameCount 300"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + overlayFrameCount: 300, + overlayEffectiveFrames: 0, + }, + }), + ).toEqual(["overlayEffectiveFrames 0 out of range for overlayFrameCount 300"]); + }); +}); + +describe("CUDA progress interval fields (module 3 surface)", () => { + it("parses the additive temporal/overlay interval counters from PROGRESS lines", () => { + expect( + parseWindowsGpuExportProgressLine( + 'PROGRESS {"currentFrame":120,"totalFrames":300,"percentage":40,"intervalTemporalBlurStationaryFrames":3,"intervalTemporalBgCacheBuilds":4,"intervalTemporalBgCacheHits":9,"intervalOverlayStaticRegionBlends":11}', + ), + ).toMatchObject({ + currentFrame: 120, + totalFrames: 300, + percentage: 40, + intervalTemporalBlurStationaryFrames: 3, + intervalTemporalBgCacheBuilds: 4, + intervalTemporalBgCacheHits: 9, + intervalOverlayStaticRegionBlends: 11, + }); + }); +}); diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index 16b2993e8..b5f7dc2b5 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -6,10 +6,18 @@ import path from "node:path"; import { performance } from "node:perf_hooks"; import type { Readable, Writable } from "node:stream"; import { promisify } from "node:util"; -import type { WebContents } from "electron"; +import type { MessagePortMain, WebContents } from "electron"; import { app, powerSaveBlocker } from "electron"; +import type { NativeStaticLayoutOverlayLayer } from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; +import { + getNativeStaticLayoutOverlayFrameByteSize, + sortNativeStaticLayoutOverlayLayers, + validateNativeStaticLayoutOverlayLayer, +} from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; import { getFfmpegBinaryPath, getFfprobeBinaryPath } from "../ffmpeg/binary"; import type { + ExportEncoderPreference, + ExportVideoCodec, NativeExportEncodingMode, NativeStaticLayoutBackend, NativeStaticLayoutExportArgsConfig, @@ -30,8 +38,8 @@ import { buildTrimmedSourceAudioFilter, createNativeSquircleMaskPgmBuffer, getEditedAudioExtension, + getNativeEncoderCandidates, getNativeVideoInputByteSize, - getPreferredNativeVideoEncoders, isNativeCudaOutOfMemory, parseAvailableFfmpegEncoders, } from "../nativeVideoExport"; @@ -42,6 +50,10 @@ const getNowMs = () => performance.now(); const formatFfmpegSeconds = (milliseconds: number) => (milliseconds / 1000).toFixed(3); const MISSING_NATIVE_STATIC_BACKGROUND_COLOR = "#ffffff"; const NATIVE_EXPORT_HIGH_PRIORITY = os.constants.priority.PRIORITY_HIGH; +// Dummy frame size used to probe whether an encoder can initialize. Must be +// large enough to satisfy hardware encoder minimums (NVENC rejects frames +// smaller than ~192x192 on recent NVIDIA drivers), while staying cheap. +const NATIVE_ENCODER_PROBE_DIMENSION = 256; const NVIDIA_PCI_VENDOR_ID = 0x10de; const NVIDIA_CUDA_EXPORT_ENV = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; const NVIDIA_CUDA_ALLOW_AUDIO_EXPORT_ENV = "RECORDLY_NVIDIA_CUDA_ALLOW_AUDIO_EXPORT"; @@ -65,6 +77,51 @@ type ElectronGpuInfoLike = { gpuDevice?: ElectronGpuDeviceLike[]; }; +export const NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION = 1; + +export type NativeVideoExportFramePortMessage = + | { + type: "hello"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + capabilityProbe: ArrayBuffer; + } + | { + type: "frame"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId: number; + sequence: number; + frame: ArrayBuffer; + }; + +export type NativeVideoExportFramePortResponse = + | { + type: "ready"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + transferable: true; + transferProbe: ArrayBuffer; + } + | { + type: "ack"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId: number; + sequence: number; + success: true; + } + | { + type: "error"; + protocol: typeof NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION; + sessionId: string; + requestId?: number; + sequence?: number; + success: false; + error: string; + fallbackAvailable: boolean; + }; + export type NativeVideoExportSession = { ffmpegProcess: ChildProcessByStdio; outputPath: string; @@ -80,10 +137,383 @@ export type NativeVideoExportSession = { completionPromise: Promise; sender: WebContents | null; pendingWriteRequestIds: Set; + framePort: MessagePortMain | null; + framePortReady: boolean; + nextFrameSequence: number; + pendingFrameRequests: Map; + completedFrameRequestIds: Set; }; export const nativeVideoExportSessions = new Map(); +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null; +} + +function isArrayBuffer(value: unknown): value is ArrayBuffer { + return value instanceof ArrayBuffer; +} + +function isValidRequestNumber(value: unknown): value is number { + return typeof value === "number" && Number.isSafeInteger(value) && value >= 0; +} + +function sendNativeVideoExportFramePortMessage( + port: MessagePortMain, + message: NativeVideoExportFramePortResponse, + transfer?: ArrayBuffer, +) { + try { + if (transfer) { + // Electron 43 types MessagePortMain's transfer list as MessagePortMain[], + // although Chromium's structured clone implementation also accepts + // ArrayBuffers. Verify detachment at runtime instead of assuming zero-copy. + port.postMessage(message, [transfer as unknown as MessagePortMain]); + return transfer.byteLength === 0; + } + port.postMessage(message); + return true; + } catch { + return false; + } +} + +export function sendNativeVideoExportFramePortError( + port: MessagePortMain, + sessionId: string, + error: string, + options: { + requestId?: number; + sequence?: number; + fallbackAvailable: boolean; + }, +) { + return sendNativeVideoExportFramePortMessage(port, { + type: "error", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + success: false, + error, + ...options, + }); +} + +function clearNativeVideoExportFramePort(session: NativeVideoExportSession) { + const port = session.framePort; + session.framePort = null; + session.framePortReady = false; + session.pendingFrameRequests.clear(); + if (port) { + try { + port.close(); + } catch { + // The renderer may already have closed the port. + } + } +} + +export function flushNativeVideoExportFramePortPendingRequests( + sessionId: string, + session: NativeVideoExportSession, + error: string, +) { + const port = session.framePort; + if (port && session.framePortReady) { + for (const [requestId, pendingRequest] of session.pendingFrameRequests) { + sendNativeVideoExportFramePortError(port, sessionId, error, { + requestId, + sequence: pendingRequest.sequence, + fallbackAvailable: false, + }); + } + } + session.pendingFrameRequests.clear(); +} + +function handleNativeVideoExportFramePortMessage( + sessionId: string, + session: NativeVideoExportSession, + port: MessagePortMain, + value: unknown, +) { + if (!isRecord(value)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Invalid native export frame message", + { + fallbackAvailable: false, + }, + ); + return; + } + + if (value.protocol !== NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Unsupported native export frame protocol", + { + fallbackAvailable: true, + }, + ); + return; + } + if (value.sessionId !== sessionId) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame session mismatch", + { + fallbackAvailable: false, + }, + ); + return; + } + + if (value.type === "hello") { + if (session.framePortReady) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel handshake was already completed", + { fallbackAvailable: false }, + ); + return; + } + if (!isArrayBuffer(value.capabilityProbe) || value.capabilityProbe.byteLength !== 1) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel transferable probe was invalid", + { fallbackAvailable: true }, + ); + clearNativeVideoExportFramePort(session); + return; + } + + if ( + !sendNativeVideoExportFramePortMessage( + port, + { + type: "ready", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + transferable: true, + transferProbe: value.capabilityProbe, + }, + value.capabilityProbe, + ) + ) { + clearNativeVideoExportFramePort(session); + return; + } + session.framePortReady = true; + return; + } + + if (value.type !== "frame") { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Unknown native export frame message", + { + fallbackAvailable: false, + }, + ); + return; + } + if (!session.framePortReady) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame channel handshake is incomplete", + { fallbackAvailable: true }, + ); + return; + } + + const requestId = value.requestId; + const sequence = value.sequence; + if (!isValidRequestNumber(requestId) || !isValidRequestNumber(sequence)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame request and sequence must be non-negative safe integers", + { fallbackAvailable: false }, + ); + return; + } + if ( + session.completedFrameRequestIds.has(requestId) || + session.pendingFrameRequests.has(requestId) + ) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Duplicate native export frame request", + { + requestId, + sequence, + fallbackAvailable: false, + }, + ); + return; + } + if (sequence !== session.nextFrameSequence) { + sendNativeVideoExportFramePortError( + port, + sessionId, + sequence < session.nextFrameSequence + ? "Duplicate native export frame sequence" + : `Out-of-order native export frame sequence; expected ${session.nextFrameSequence}`, + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (!isArrayBuffer(value.frame) || value.frame.byteLength === 0) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native export frame payload must be a non-empty ArrayBuffer", + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (session.inputMode !== "h264-stream" && value.frame.byteLength !== session.inputByteSize) { + sendNativeVideoExportFramePortError( + port, + sessionId, + `Native video export expected ${session.inputByteSize} bytes per frame but received ${value.frame.byteLength}`, + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + if (session.terminating) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session was cancelled", + { requestId, sequence, fallbackAvailable: false }, + ); + return; + } + + session.pendingFrameRequests.set(requestId, { sequence }); + session.completedFrameRequestIds.add(requestId); + session.nextFrameSequence += 1; + void enqueueNativeVideoExportFrameWrite(session, value.frame) + .then(() => { + const pendingRequest = session.pendingFrameRequests.get(requestId); + if (!pendingRequest) { + return; + } + session.pendingFrameRequests.delete(requestId); + if ( + !sendNativeVideoExportFramePortMessage(port, { + type: "ack", + protocol: NATIVE_VIDEO_EXPORT_FRAME_PROTOCOL_VERSION, + sessionId, + requestId, + sequence: pendingRequest.sequence, + success: true, + }) + ) { + clearNativeVideoExportFramePort(session); + } + }) + .catch((error: unknown) => { + const nativeError = error instanceof Error ? error : new Error(String(error)); + session.stdinError = nativeError; + const pendingRequest = session.pendingFrameRequests.get(requestId); + if (!pendingRequest) { + return; + } + session.pendingFrameRequests.delete(requestId); + if ( + !sendNativeVideoExportFramePortError(port, sessionId, nativeError.message, { + requestId, + sequence: pendingRequest.sequence, + fallbackAvailable: false, + }) + ) { + clearNativeVideoExportFramePort(session); + } + }); +} + +export function attachNativeVideoExportFramePort( + sessionId: string, + session: NativeVideoExportSession, + port: MessagePortMain, + sender: WebContents, +) { + if (session.terminating) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session was cancelled", + { + fallbackAvailable: false, + }, + ); + port.close(); + return false; + } + + if (session.framePort) { + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + "Native export frame channel was replaced", + ); + clearNativeVideoExportFramePort(session); + } + + session.sender = sender; + session.framePort = port; + session.framePortReady = false; + session.nextFrameSequence = 0; + session.pendingFrameRequests.clear(); + session.completedFrameRequestIds.clear(); + port.on("message", (event) => { + if (session.framePort !== port) { + return; + } + handleNativeVideoExportFramePortMessage(sessionId, session, port, event.data); + }); + port.on("close", () => { + if (session.framePort !== port) { + return; + } + session.framePort = null; + session.framePortReady = false; + session.pendingFrameRequests.clear(); + }); + sender.once("destroyed", () => { + if (session.framePort !== port) { + return; + } + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + "Native export renderer was reloaded before frame acknowledgements settled", + ); + clearNativeVideoExportFramePort(session); + }); + port.start(); + return true; +} + +export function closeNativeVideoExportFramePort( + sessionId: string, + session: NativeVideoExportSession, + error: string, +) { + flushNativeVideoExportFramePortPendingRequests(sessionId, session, error); + clearNativeVideoExportFramePort(session); +} + export interface NativeStaticLayoutTimelineSegment { sourceStartMs: number; sourceEndMs: number; @@ -95,12 +525,16 @@ export interface NativeStaticLayoutTimelineSegment { export interface NativeStaticLayoutExportOptions { sessionId?: string; inputPath: string; + /** High-level output contract; encoder names never cross this boundary. */ + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; width: number; height: number; frameRate: number; bitrate: number; encodingMode: NativeExportEncodingMode; durationSec: number; + overlayLayers?: NativeStaticLayoutOverlayLayer[]; contentWidth: number; contentHeight: number; offsetX: number; @@ -145,8 +579,32 @@ export interface NativeStaticLayoutExportOptions { aspectRatio: number; }>; cursorAtlasMetadataPath?: string | null; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; zoomTelemetryPath?: string | null; + /** Resolved temporal zoom motion blur plan (temporalMotionBlur.ts config). */ + temporalBlur?: { + sampleCount: number; + shutterFraction: number; + weightCurvePower: number; + } | null; + /** JSON manifest describing renderer-prepared RGBA overlay sidecars. */ + overlayManifestPath?: string | null; + /** + * True when the renderer excluded cursor pixels from the overlay sidecars + * and the native cursor atlas owns them. The CUDA compositor must draw the + * atlas on top of the sidecars; stripping the cursor assets would drop the + * cursor, and passing them while the sidecar still baked them would + * double-render. Absent/false keeps the baked-sidecar contract. + */ + cursorAtlasOwned?: boolean; timelineSegments?: NativeStaticLayoutTimelineSegment[]; timelineMapPath?: string | null; chunkDurationSec?: number; @@ -163,6 +621,14 @@ export interface NativeStaticLayoutExportProgress { elapsedMs?: number; averageFps?: number; instantFps?: number; + /** + * End-to-end estimate (frames / wall-clock since process spawn). This + * includes preparation time, so it is NOT a native encode speed; it must + * never be presented as measured encode FPS. + */ + estimatedFps?: number; + /** Where the reported FPS values came from. */ + fpsSource?: "native" | "estimated"; intervalMs?: number; intervalFrames?: number; intervalDecodeWallMs?: number; @@ -176,6 +642,11 @@ export interface NativeStaticLayoutExportProgress { intervalRoiCompositeFrames?: number; intervalMonolithicCompositeFrames?: number; intervalCopyCompositeFrames?: number; + intervalZoomBlurFrames?: number; + intervalTemporalBlurStationaryFrames?: number; + intervalTemporalBgCacheBuilds?: number; + intervalTemporalBgCacheHits?: number; + intervalOverlayStaticRegionBlends?: number; currentFrame: number; totalFrames: number; percentage: number; @@ -231,9 +702,85 @@ export interface WindowsGpuExportSummary { realtimeMultiplier?: number; } +export interface NvidiaCudaNativeSummary { + success?: boolean; + selectionStage?: string; + sourceTimestampMode?: string; + timelineMode?: string; + frames?: number; + totalMs?: number; + fps?: number; + measuredFps?: number; + mappedDisplayFrames?: number; + selectedDisplayFrames?: number; + skippedDisplayFrames?: number; + roiCompositeFrames?: number; + monolithicCompositeFrames?: number; + copyCompositeFrames?: number; + cursorAtlas?: boolean; + webcamOverlay?: boolean; + zoomOverlay?: boolean; + zoomSamples?: number; + overlayLayers?: number; + overlayBlendFrames?: number; + /** Renderer-resolved temporal zoom motion blur sample count (config, not + * measured: the helper does not yet echo it in its summary). */ + temporalBlurSampleCount?: number; + /** Temporal blur output frames (additive compositor counter). */ + temporalBlurFrames?: number; + /** Total temporal blur sample composites across all output frames. */ + temporalBlurSamplesTotal?: number; + /** Temporal blur frames that reused the precomposed invariant background. */ + temporalBlurBgPrecomposedFrames?: number; + /** Overlay sidecar frames read from disk (additive). */ + overlayFileLoads?: number; + /** Overlay sidecar frames served from the resident ring slot (additive). */ + overlayCacheHits?: number; + /** Overlay blends confined to static alpha bounds (additive). */ + overlayStaticRegionBlends?: number; + /** Overlay sidecar width in pixels (renderer-derived, additive). */ + overlayWidth?: number; + /** Overlay sidecar height in pixels (renderer-derived, additive). */ + overlayHeight?: number; + /** Overlay sidecar logical frame count (output duration frames). */ + overlayFrameCount?: number; + /** Overlay sidecar physical frame count (effectiveFrameCount when deduped). */ + overlayPhysicalFrames?: number; + /** Overlay sidecar stored frame count after identical-suffix dedup. */ + overlayEffectiveFrames?: number; + /** Temporal blur frames composited by the stationary fused kernel. */ + temporalBlurStationaryFrames?: number; + /** Temporal blur invariant-background cache builds (additive). */ + temporalBgCacheBuilds?: number; + /** Temporal blur invariant-background cache hits (additive). */ + temporalBgCacheHits?: number; + /** + * Overlay host-read wall time in ms. The helper currently reports only the + * combined overlayUploadMs; these fields stay absent until main.cu separates + * host reads from H2D enqueues. + */ + overlayHostReadMs?: number; + overlayH2DEnqueueMs?: number; + /** Stage wall/GPU timings in ms, additive over the encode run. */ + compositeMs?: number; + compositeGpuMs?: number; + zoomBlurGpuMs?: number; + overlayBlendGpuMs?: number; + overlayUploadMs?: number; + nvencMs?: number; + packetWriteMs?: number; + decodeMs?: number; + decodeWallMs?: number; + encodeMs?: number; + flushMs?: number; + realtimeMultiplier?: number; + outputBytes?: number; +} + export interface NvidiaCudaExportSummary { success?: boolean; inputPath?: string; + outputCodec?: ExportVideoCodec; outputPath?: string; fps?: number; bitrateMbps?: number; @@ -252,26 +799,7 @@ export interface NvidiaCudaExportSummary { mux?: number; endToEnd?: number; }; - nativeSummary?: { - success?: boolean; - selectionStage?: string; - sourceTimestampMode?: string; - timelineMode?: string; - frames?: number; - totalMs?: number; - fps?: number; - measuredFps?: number; - mappedDisplayFrames?: number; - selectedDisplayFrames?: number; - skippedDisplayFrames?: number; - roiCompositeFrames?: number; - monolithicCompositeFrames?: number; - copyCompositeFrames?: number; - cursorAtlas?: boolean; - webcamOverlay?: boolean; - zoomOverlay?: boolean; - zoomSamples?: number; - }; + nativeSummary?: NvidiaCudaNativeSummary; nativeProcessPriorityBoosted?: boolean; appRuntimeGuard?: { powerGuardStarted?: boolean; @@ -327,6 +855,15 @@ export interface NativeStaticLayoutExportMetrics extends NativeVideoAudioMuxMetr chunks: NativeStaticLayoutChunkMetric[]; } +export interface NativeStaticLayoutExportResult { + outputPath: string; + metrics: NativeStaticLayoutExportMetrics; + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + encoderName: string; + route: NativeStaticLayoutBackend; +} + export interface NativeStaticLayoutExportSession { terminating: boolean; currentProcess: ReturnType | null; @@ -335,6 +872,11 @@ export interface NativeStaticLayoutExportSession { export function cleanupNativeVideoExportSessions() { for (const [sessionId, session] of nativeVideoExportSessions) { session.terminating = true; + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export sessions were cleaned up", + ); try { if (!session.ffmpegProcess.stdin.destroyed) { session.ffmpegProcess.stdin.destroy(); @@ -397,6 +939,277 @@ export function parseNvidiaCudaExportSummary(stdout: string): NvidiaCudaExportSu } } +const NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS = [ + "temporalBlurSampleCount", + "temporalBlurSamplesTotal", + "temporalBlurBgPrecomposedFrames", + "temporalBlurFrames", + "temporalBlurStationaryFrames", + "temporalBgCacheBuilds", + "temporalBgCacheHits", + "compositeGpuMs", + "zoomBlurGpuMs", + "overlayBlendGpuMs", + "overlayUploadMs", + "overlayFileLoads", + "overlayCacheHits", + "overlayStaticRegionBlends", + "overlayBlendFrames", + "overlayWidth", + "overlayHeight", + "overlayFrameCount", + "overlayPhysicalFrames", + "overlayEffectiveFrames", + "overlayHostReadMs", + "overlayH2DEnqueueMs", + "nvencMs", + "packetWriteMs", + "totalMs", + "encodeMs", + "decodeMs", + "decodeWallMs", + "compositeMs", + "flushMs", + "realtimeMultiplier", + "outputBytes", +] as const satisfies ReadonlyArray; + +export type NvidiaCudaNativeSummaryMetricField = + (typeof NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS)[number]; + +/** + * Maps the additive native compositor counters into a flat metric object for + * the completion log. Only fields the helper actually reported as finite + * numbers are included, so absent counters never appear as undefined noise and + * non-finite payloads are never surfaced as measured values. The values are + * cumulative over the whole helper run (decode/compose/overlay/NVENC stages), + * never interval deltas. + */ +export function resolveNvidiaCudaNativeSummaryMetrics( + nativeSummary: NvidiaCudaNativeSummary | undefined, +): Partial> { + const metrics: Partial> = {}; + if (!nativeSummary) { + return metrics; + } + + for (const field of NVIDIA_CUDA_NATIVE_SUMMARY_METRIC_FIELDS) { + const value = nativeSummary[field]; + if (typeof value === "number" && Number.isFinite(value)) { + metrics[field] = value; + } + } + return metrics; +} + +export type NvidiaCudaOverlaySidecarSummaryMetric = Pick< + NvidiaCudaNativeSummary, + | "overlayWidth" + | "overlayHeight" + | "overlayFrameCount" + | "overlayPhysicalFrames" + | "overlayEffectiveFrames" +>; + +/** + * Derives the overlay sidecar dimensions and physical/effective frame counts + * from the renderer-prepared overlay layers for CUDA summary surfacing. All + * fields are additive and only present when overlay layers exist. The physical + * count is the sidecar's stored frame count (effectiveFrameCount after + * identical-suffix dedup); the logical count is the output duration. + */ +export function resolveNvidiaCudaOverlaySidecarSummaryMetrics( + options: NativeStaticLayoutExportOptions, +): Partial { + const layer = options.overlayLayers?.[0]; + if ( + !layer || + !Number.isFinite(layer.width) || + !Number.isFinite(layer.height) || + !Number.isFinite(layer.frameCount) + ) { + return {}; + } + + const metrics: Partial = { + overlayWidth: Math.max(1, Math.round(layer.width)), + overlayHeight: Math.max(1, Math.round(layer.height)), + overlayFrameCount: Math.max(1, Math.round(layer.frameCount)), + overlayPhysicalFrames: Math.max( + 1, + Math.round(layer.effectiveFrameCount ?? layer.frameCount), + ), + }; + if (layer.effectiveFrameCount !== undefined) { + metrics.overlayEffectiveFrames = Math.max(1, Math.round(layer.effectiveFrameCount)); + } + return metrics; +} + +/** + * Resolves the measured native encode FPS reported by the helper: the flush + * span measuredFps is authoritative, with the encode-loop fps as the backward- + * compatible fallback. Never derives FPS from wall-clock estimates. + */ +export function resolveNvidiaCudaNativeFps( + summary: NvidiaCudaExportSummary | undefined, +): number | undefined { + const nativeSummary = summary?.nativeSummary; + const measuredFps = nativeSummary?.measuredFps; + if (typeof measuredFps === "number" && Number.isFinite(measuredFps) && measuredFps > 0) { + return measuredFps; + } + const fallbackFps = nativeSummary?.fps; + if (typeof fallbackFps === "number" && Number.isFinite(fallbackFps) && fallbackFps > 0) { + return fallbackFps; + } + return undefined; +} + +/** + * Verifies the additive compositor counters against each other so the + * completion diagnostics can prove (rather than assume) that the reported + * stage timings are sane. Returns human-readable issue strings; an empty array + * means the metrics are internally consistent. These are diagnostics only and + * never gate the export result. + */ +export function validateNvidiaCudaStageMetricInvariants( + summary: NvidiaCudaExportSummary, +): string[] { + const issues: string[] = []; + const native = summary.nativeSummary; + if (!native) { + return issues; + } + + const totalMs = native.totalMs; + if (typeof totalMs === "number" && Number.isFinite(totalMs) && totalMs >= 0) { + const stageFields: ReadonlyArray< + readonly [NvidiaCudaNativeSummaryMetricField, number | undefined] + > = [ + ["compositeGpuMs", native.compositeGpuMs], + ["overlayBlendGpuMs", native.overlayBlendGpuMs], + ["overlayUploadMs", native.overlayUploadMs], + ["nvencMs", native.nvencMs], + ["packetWriteMs", native.packetWriteMs], + ["compositeMs", native.compositeMs], + ["zoomBlurGpuMs", native.zoomBlurGpuMs], + ["encodeMs", native.encodeMs], + ["decodeWallMs", native.decodeWallMs], + ]; + for (const [field, value] of stageFields) { + if (typeof value === "number" && Number.isFinite(value) && value > totalMs + 0.5) { + issues.push(`${field} ${value}ms exceeds helper wall time ${totalMs}ms`); + } + } + } + + const temporalBlurFrames = native.temporalBlurFrames; + if (typeof temporalBlurFrames === "number" && Number.isFinite(temporalBlurFrames)) { + const samplesTotal = native.temporalBlurSamplesTotal; + if ( + typeof samplesTotal === "number" && + Number.isFinite(samplesTotal) && + samplesTotal < temporalBlurFrames + ) { + issues.push( + `temporalBlurSamplesTotal ${samplesTotal} below temporalBlurFrames ${temporalBlurFrames}`, + ); + } + const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; + if ( + typeof bgPrecomposedFrames === "number" && + Number.isFinite(bgPrecomposedFrames) && + bgPrecomposedFrames > temporalBlurFrames + ) { + issues.push( + `temporalBlurBgPrecomposedFrames ${bgPrecomposedFrames} exceeds temporalBlurFrames ${temporalBlurFrames}`, + ); + } + } + + const stationaryFrames = native.temporalBlurStationaryFrames; + if ( + typeof stationaryFrames === "number" && + Number.isFinite(stationaryFrames) && + typeof temporalBlurFrames === "number" && + Number.isFinite(temporalBlurFrames) && + stationaryFrames > temporalBlurFrames + ) { + issues.push( + `temporalBlurStationaryFrames ${stationaryFrames} exceeds temporalBlurFrames ${temporalBlurFrames}`, + ); + } + const bgCacheBuilds = native.temporalBgCacheBuilds; + const bgCacheHits = native.temporalBgCacheHits; + if ( + typeof bgCacheBuilds === "number" && + Number.isFinite(bgCacheBuilds) && + typeof bgCacheHits === "number" && + Number.isFinite(bgCacheHits) + ) { + const bgCacheTotal = bgCacheBuilds + bgCacheHits; + const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; + if ( + typeof bgPrecomposedFrames === "number" && + Number.isFinite(bgPrecomposedFrames) && + bgCacheTotal > bgPrecomposedFrames + ) { + issues.push( + `temporalBgCacheBuilds ${bgCacheBuilds} + temporalBgCacheHits ${bgCacheHits} exceed temporalBlurBgPrecomposedFrames ${bgPrecomposedFrames}`, + ); + } else if ( + typeof temporalBlurFrames === "number" && + Number.isFinite(temporalBlurFrames) && + bgCacheTotal > temporalBlurFrames + ) { + issues.push( + `temporalBgCacheBuilds ${bgCacheBuilds} + temporalBgCacheHits ${bgCacheHits} exceed temporalBlurFrames ${temporalBlurFrames}`, + ); + } + } + + const overlayBlendFrames = native.overlayBlendFrames; + const overlayStaticRegionBlends = native.overlayStaticRegionBlends; + if ( + typeof overlayStaticRegionBlends === "number" && + Number.isFinite(overlayStaticRegionBlends) && + typeof overlayBlendFrames === "number" && + Number.isFinite(overlayBlendFrames) && + overlayStaticRegionBlends > overlayBlendFrames + ) { + issues.push( + `overlayStaticRegionBlends ${overlayStaticRegionBlends} exceeds overlayBlendFrames ${overlayBlendFrames}`, + ); + } + + const overlayFrameCount = native.overlayFrameCount; + if (typeof overlayFrameCount === "number" && Number.isFinite(overlayFrameCount)) { + const overlayPhysicalFrames = native.overlayPhysicalFrames; + if ( + typeof overlayPhysicalFrames === "number" && + Number.isFinite(overlayPhysicalFrames) && + overlayPhysicalFrames > overlayFrameCount + ) { + issues.push( + `overlayPhysicalFrames ${overlayPhysicalFrames} exceeds overlayFrameCount ${overlayFrameCount}`, + ); + } + const overlayEffectiveFrames = native.overlayEffectiveFrames; + if ( + typeof overlayEffectiveFrames === "number" && + Number.isFinite(overlayEffectiveFrames) && + (overlayEffectiveFrames < 1 || overlayEffectiveFrames > overlayFrameCount) + ) { + issues.push( + `overlayEffectiveFrames ${overlayEffectiveFrames} out of range for overlayFrameCount ${overlayFrameCount}`, + ); + } + } + + return issues; +} + function getFiniteNumber(value: unknown) { const numberValue = typeof value === "string" ? Number(value) : value; return typeof numberValue === "number" && Number.isFinite(numberValue) ? numberValue : null; @@ -416,6 +1229,7 @@ export function validateNvidiaCudaExportSummary( durationSec: number; targetFrames: number; requiresTimelineSync?: boolean; + videoCodec?: ExportVideoCodec; }, ) { const issues: string[] = []; @@ -428,6 +1242,11 @@ export function validateNvidiaCudaExportSummary( const outputVideoDurationSec = getNvidiaCudaOutputStreamNumber(summary.outputVideo, "duration"); const outputAudioDurationSec = getNvidiaCudaOutputStreamNumber(summary.outputAudio, "duration"); + if (expected.videoCodec && summary.outputCodec !== expected.videoCodec) { + issues.push( + `CUDA output codec ${summary.outputCodec ?? "unknown"} does not match expected ${expected.videoCodec}`, + ); + } if (!summary.outputVideo) { issues.push("missing output video probe"); } @@ -807,6 +1626,11 @@ export function parseWindowsGpuExportProgressLine( intervalRoiCompositeFrames?: unknown; intervalMonolithicCompositeFrames?: unknown; intervalCopyCompositeFrames?: unknown; + intervalZoomBlurFrames?: unknown; + intervalTemporalBlurStationaryFrames?: unknown; + intervalTemporalBgCacheBuilds?: unknown; + intervalTemporalBgCacheHits?: unknown; + intervalOverlayStaticRegionBlends?: unknown; stage?: unknown; }; const currentFrame = Number(parsed.currentFrame); @@ -845,6 +1669,11 @@ export function parseWindowsGpuExportProgressLine( "intervalRoiCompositeFrames", "intervalMonolithicCompositeFrames", "intervalCopyCompositeFrames", + "intervalZoomBlurFrames", + "intervalTemporalBlurStationaryFrames", + "intervalTemporalBgCacheBuilds", + "intervalTemporalBgCacheHits", + "intervalOverlayStaticRegionBlends", ] as const; for (const field of optionalNumberFields) { const value = Number(parsed[field]); @@ -870,6 +1699,47 @@ export function mapNvidiaCudaWrapperProgressPercentage(progress: NativeStaticLay return progress.percentage; } +// Distinguish native measured encode FPS from the end-to-end (preparation- +// inclusive) estimate. Only averageFps/instantFps reported by the native helper +// count as measured encode speed; the frames/wall-clock estimate since process +// spawn must be surfaced separately so callers never mistake it for encode +// throughput. +export function resolveNativeStaticLayoutFpsFields( + progress: NativeStaticLayoutExportProgress, + elapsedMs: number, +): { + averageFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; +} { + const nativeAverageFps = + typeof progress.averageFps === "number" && + Number.isFinite(progress.averageFps) && + progress.averageFps > 0 + ? progress.averageFps + : undefined; + const nativeInstantFps = + typeof progress.instantFps === "number" && + Number.isFinite(progress.instantFps) && + progress.instantFps > 0 + ? progress.instantFps + : undefined; + const hasNativeMeasured = nativeInstantFps !== undefined || nativeAverageFps !== undefined; + const estimatedFps = + !hasNativeMeasured && elapsedMs > 0 && progress.currentFrame > 0 + ? (progress.currentFrame * 1000) / elapsedMs + : undefined; + return { + averageFps: nativeAverageFps, + estimatedFps, + fpsSource: hasNativeMeasured + ? "native" + : estimatedFps !== undefined + ? "estimated" + : undefined, + }; +} + export function hasNativeStaticLayoutProgressAdvanced( progress: { currentFrame: number; percentage: number; stage?: string }, previous: { currentFrame: number; percentage: number; stage?: string }, @@ -1491,6 +2361,7 @@ export function flushNativeVideoExportPendingWriteRequests( session: NativeVideoExportSession, error: string, ) { + flushNativeVideoExportFramePortPendingRequests(sessionId, session, error); for (const requestId of session.pendingWriteRequestIds) { sendNativeVideoExportWriteFrameResult(session.sender, sessionId, requestId, { success: false, @@ -2075,15 +2946,38 @@ export async function getNativeExportCapabilities(): Promise, +) { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + if (encoderPreference === "cpu") { + return "encoder-preference-cpu-requires-native-rawvideo"; + } + if (encoderPreference === "hardware" && videoCodec === "h264") { + return "encoder-preference-hardware-requires-native-rawvideo"; + } + return null; +} + function getNvidiaCudaBitrateMbps(options: NativeStaticLayoutExportOptions) { return Math.max(1, Math.round(options.bitrate / 1_000_000)); } @@ -2196,6 +3104,11 @@ export function buildExperimentalWindowsGpuStaticLayoutArgs( options: NativeStaticLayoutExportOptions, outputPath: string, ) { + if (options.videoCodec === "hevc") { + throw new Error( + "HEVC native static layout requires the generalized NVIDIA CUDA compositor", + ); + } const shadowPixels = Math.round(clampUnit(options.shadowIntensity ?? 0) * 64); const backgroundBlurPx = Math.max(0, options.backgroundBlurPx ?? 0); const pixelCount = options.width * options.height; @@ -2404,6 +3317,39 @@ async function prepareWindowsGpuCursorAtlas( return { atlasPath, metadataPath }; } +/** + * Resolves the native cursor asset paths that may be handed to a GPU compositor + * wrapper. The Windows GPU compositor prep writes CSV telemetry/atlas artifacts + * onto the options; the NVIDIA CUDA wrapper's `--cursor-json` contract is a JSON + * {"samples":[...]} payload (the pipeline rejects raw CSV/TSV rows), so a CSV + * path must never reach it. When overlay sidecars are present AND the cursor is + * baked into the transparent RGBA layer (cursorAtlasOwned is not true), the CUDA + * route must not draw the cursor again: stripping the assets is mandatory, never + * a silent degradation (the wrapper is never given a malformed cursor file and + * never double-renders the baked cursor). When cursorAtlasOwned is true the + * sidecar excluded cursor pixels, so the assets must pass through untouched. + */ +export function resolveNvidiaCudaCursorAssets( + options: NativeStaticLayoutExportOptions, + strip: boolean, +): Pick< + NativeStaticLayoutExportOptions, + "cursorTelemetryPath" | "cursorAtlasPath" | "cursorAtlasMetadataPath" +> { + if (!strip) { + return { + cursorTelemetryPath: options.cursorTelemetryPath ?? null, + cursorAtlasPath: options.cursorAtlasPath ?? null, + cursorAtlasMetadataPath: options.cursorAtlasMetadataPath ?? null, + }; + } + return { + cursorTelemetryPath: null, + cursorAtlasPath: null, + cursorAtlasMetadataPath: null, + }; +} + async function prepareNvidiaCudaCursorTelemetry( options: NativeStaticLayoutExportOptions, outputPath: string, @@ -2489,16 +3435,10 @@ async function prepareNvidiaCudaCursorAtlas( return { atlasPath, metadataPath }; } -async function prepareWindowsGpuZoomTelemetry( - options: NativeStaticLayoutExportOptions, - outputPath: string, -) { - const telemetry = options.zoomTelemetry; - if (!telemetry || telemetry.length === 0) { - return null; - } - - const lines = telemetry +export function formatNativeStaticLayoutZoomTelemetryLines( + telemetry: NonNullable, +): string[] { + return telemetry .filter((sample) => { return ( Number.isFinite(sample.timeMs) && @@ -2510,13 +3450,33 @@ async function prepareWindowsGpuZoomTelemetry( .map((sample) => { const timeMs = Math.max(0, sample.timeMs); const scale = Math.max(0.01, sample.scale); + const blurStrength = Number.isFinite(sample.blurStrength) + ? Math.max(0, sample.blurStrength ?? 0) + : 0; + const blurCenterX = Number.isFinite(sample.blurCenterX) ? (sample.blurCenterX ?? 0) : 0; + const blurCenterY = Number.isFinite(sample.blurCenterY) ? (sample.blurCenterY ?? 0) : 0; return [ formatCliNumber(timeMs), formatCliNumber(scale), formatCliNumber(sample.x), formatCliNumber(sample.y), + formatCliNumber(blurStrength), + formatCliNumber(blurCenterX), + formatCliNumber(blurCenterY), ].join(","); }); +} + +async function prepareWindowsGpuZoomTelemetry( + options: NativeStaticLayoutExportOptions, + outputPath: string, +) { + const telemetry = options.zoomTelemetry; + if (!telemetry || telemetry.length === 0) { + return null; + } + + const lines = formatNativeStaticLayoutZoomTelemetryLines(telemetry); if (lines.length === 0) { return null; @@ -2645,6 +3605,42 @@ async function prepareNativeStaticLayoutSourceInput( }; } +export function buildNativeStaticLayoutOverlayManifest( + layers: readonly NativeStaticLayoutOverlayLayer[], +) { + return { + layers: [...layers] + .sort((left, right) => left.order - right.order || left.id.localeCompare(right.id)) + .map((layer) => ({ + id: layer.id, + path: layer.path, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + // frameCount stays the logical output duration; effectiveFrameCount + // is the physical frame count the renderer wrote when identical-suffix + // dedup truncated the sidecar. Absent when every frame differs. + frameCount: layer.frameCount, + ...(layer.effectiveFrameCount !== undefined + ? { effectiveFrameCount: layer.effectiveFrameCount } + : {}), + })), + }; +} + +export function getNativeStaticLayoutOverlayExpectedSidecarBytes( + layer: NativeStaticLayoutOverlayLayer, +) { + // Physical sidecar byte-size validation must use the physical frame count + // (effectiveFrameCount when present), never the logical output duration + // (frameCount), so deduped overlays are not rejected as truncated. + const physicalFrameCount = layer.effectiveFrameCount ?? layer.frameCount; + return ( + getNativeStaticLayoutOverlayFrameByteSize(layer.width, layer.height) * physicalFrameCount + ); +} + export function buildExperimentalNvidiaCudaStaticLayoutArgs( options: NativeStaticLayoutExportOptions, outputPath: string, @@ -2670,6 +3666,8 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs( String(Math.max(1, Math.round(options.frameRate))), "--bitrate-mbps", String(getNvidiaCudaBitrateMbps(options)), + "--output-codec", + options.videoCodec ?? "h264", "--encoding-mode", options.encodingMode, "--duration-sec", @@ -2772,6 +3770,19 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs( if (options.zoomTelemetryPath) { args.push("--zoom-telemetry", options.zoomTelemetryPath); } + if (options.temporalBlur && options.temporalBlur.sampleCount >= 3) { + args.push( + "--temporal-blur-sample-count", + String(Math.round(options.temporalBlur.sampleCount)), + "--temporal-blur-shutter-fraction", + formatCliNumber(options.temporalBlur.shutterFraction), + "--temporal-blur-weight-power", + formatCliNumber(options.temporalBlur.weightCurvePower), + ); + } + if (options.overlayManifestPath) { + args.push("--overlay-manifest", options.overlayManifestPath); + } if (options.timelineMapPath) { args.push("--timeline-map", options.timelineMapPath); } @@ -2810,9 +3821,36 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( const nodeCommand = resolveExperimentalNvidiaCudaNodeCommand(); const workDir = path.join(chunkDirectory, "nvidia-cuda-work"); + let effectiveOptions = options; + if (options.overlayLayers?.length) { + const overlayManifestPath = path.join(chunkDirectory, "overlay-manifest.json"); + await fs.writeFile( + overlayManifestPath, + JSON.stringify(buildNativeStaticLayoutOverlayManifest(options.overlayLayers)), + "utf8", + ); + effectiveOptions = { + ...options, + overlayManifestPath, + }; + if (options.cursorAtlasOwned !== true) { + // Overlay sidecars already contain the renderer-baked cursor; the + // Windows GPU prep may have left CSV telemetry/atlas paths on the + // options and the CUDA wrapper JSON.parses --cursor-json (a CSV file + // crashes it). Strip the cursor assets so the wrapper is never handed + // a malformed cursor file and never double-renders the baked cursor. + // When cursorAtlasOwned is true the sidecar excluded cursor pixels, so + // the prepared JSON telemetry/atlas assets pass through untouched and + // the wrapper draws the cursor natively. + effectiveOptions = { + ...effectiveOptions, + ...resolveNvidiaCudaCursorAssets(effectiveOptions, true), + }; + } + } const args = [ scriptPath, - ...buildExperimentalNvidiaCudaStaticLayoutArgs(options, outputPath, workDir), + ...buildExperimentalNvidiaCudaStaticLayoutArgs(effectiveOptions, outputPath, workDir), ]; const startedAt = getNowMs(); const startedAtIso = new Date().toISOString(); @@ -2907,14 +3945,18 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( continue; } const elapsedMs = Math.max(0, getNowMs() - startedAt); - const averageFps = - typeof progress.averageFps === "number" && - Number.isFinite(progress.averageFps) && - progress.averageFps > 0 - ? progress.averageFps - : elapsedMs > 0 && progress.currentFrame > 0 - ? (progress.currentFrame * 1000) / elapsedMs - : undefined; + const fpsFields = resolveNativeStaticLayoutFpsFields(progress, elapsedMs); + if (fpsFields.fpsSource === "estimated") { + console.warn( + "[native-static-layout-export] Native helper has not reported measured encode FPS; using preparation-inclusive estimate", + { + backend: "nvidia-cuda-compositor", + estimatedFps: fpsFields.estimatedFps, + currentFrame: progress.currentFrame, + elapsedMs, + }, + ); + } const mappedPercentage = mapNvidiaCudaWrapperProgressPercentage(progress); lastProgressPercentage = Math.max(lastProgressPercentage, mappedPercentage); const progressForStallGuard = { @@ -2937,7 +3979,7 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( sessionId: options.sessionId, backend: "nvidia-cuda-compositor", elapsedMs, - averageFps, + ...fpsFields, }); } }); @@ -3111,15 +4153,13 @@ async function runExperimentalWindowsGpuStaticLayoutExport( armStallTimeout(); } const elapsedMs = Math.max(0, getNowMs() - startedAt); + const fpsFields = resolveNativeStaticLayoutFpsFields(progress, elapsedMs); onProgress?.({ ...progress, sessionId: options.sessionId, backend: "windows-d3d11-compositor", elapsedMs, - averageFps: - elapsedMs > 0 && progress.currentFrame > 0 - ? (progress.currentFrame * 1000) / elapsedMs - : undefined, + ...fpsFields, }); } }); @@ -3190,14 +4230,79 @@ export async function exportNativeStaticLayoutVideo( ffmpegPath: string, options: NativeStaticLayoutExportOptions, onProgress?: (progress: NativeStaticLayoutExportProgress) => void, -) { +): Promise { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + if (videoCodec !== "h264" && videoCodec !== "hevc") { + throw new Error(`Unsupported native static-layout video codec: ${String(videoCodec)}`); + } + if ( + encoderPreference !== "auto" && + encoderPreference !== "hardware" && + encoderPreference !== "cpu" + ) { + throw new Error( + `Unsupported native static-layout encoder preference: ${String(encoderPreference)}`, + ); + } + options = { ...options, videoCodec, encoderPreference }; + const rawFrameFallbackReason = getNativeStaticLayoutRawFrameFallbackReason(options); + if (rawFrameFallbackReason) { + throw new Error( + `Native static-layout export requires the native rawvideo route: ${rawFrameFallbackReason}`, + ); + } + if (options.width % 2 !== 0 || options.height % 2 !== 0) { throw new Error("Native static layout export requires even output dimensions"); } if (!Number.isFinite(options.durationSec) || options.durationSec <= 0) { throw new Error("Native static layout export requires a positive duration"); } + if (options.overlayLayers?.length) { + for (const layer of sortNativeStaticLayoutOverlayLayers(options.overlayLayers)) { + const validationError = validateNativeStaticLayoutOverlayLayer(layer, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }); + if (validationError) { + throw new Error(`Invalid native overlay layer: ${validationError}`); + } + const stat = await fs.stat(layer.path); + const expectedBytes = getNativeStaticLayoutOverlayExpectedSidecarBytes(layer); + if (stat.size < expectedBytes) { + throw new Error( + `Native overlay layer ${layer.id} is truncated: expected ${expectedBytes} bytes, received ${stat.size}`, + ); + } + } + } + if ( + options.cursorAtlasOwned === true && + !( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + (options.experimentalNvidiaCudaExport === true || isExplicitNvidiaCudaExportEnabled()) + ) + ) { + // The renderer excluded cursor pixels from the overlay sidecar because + // the native atlas owns them. Only the generalized NVIDIA CUDA compositor + // can draw that atlas on top of the sidecars; the FFmpeg overlay route + // and the D3D11 helper cannot, so refusing the fallback is the only way + // to avoid silently dropping the cursor. + throw new Error( + "Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor on Windows; the FFmpeg overlay route cannot draw the cursor and the overlay sidecar excluded it.", + ); + } options = await normalizeNativeStaticLayoutBackground(options); + const nativeVideoEncoder = await resolveNativeVideoEncoder( + ffmpegPath, + options.encodingMode, + videoCodec, + encoderPreference, + ); if ( options.webcamInputPath && !(options.experimentalWindowsGpuCompositor && process.platform === "win32") @@ -3284,6 +4389,7 @@ export async function exportNativeStaticLayoutVideo( const fullConfig: NativeStaticLayoutExportArgsConfig = { inputPath: options.inputPath, outputPath: videoOnlyPath, + videoCodec, width: options.width, height: options.height, frameRate: options.frameRate, @@ -3303,12 +4409,23 @@ export async function exportNativeStaticLayoutVideo( borderRadius: options.borderRadius, shadowIntensity: options.shadowIntensity, durationSec: options.durationSec, + overlayLayers: options.overlayLayers, + videoEncoder: nativeVideoEncoder, }; const usePrecompositedLayout = shouldUsePrecompositedStaticLayout(options); let didRenderVideo = false; let didMuxAudioInline = false; - if (options.experimentalWindowsGpuCompositor && process.platform === "win32") { + if ( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + // The generalized NVIDIA CUDA compositor composites renderer-prepared + // overlay sidecars natively; the Windows D3D11 helper cannot, so it is + // skipped below when overlay layers are present. + (options.overlayLayers?.length === 0 || + options.experimentalNvidiaCudaExport === true || + isExplicitNvidiaCudaExportEnabled()) + ) { try { if (session.terminating) { throw new Error("Native static layout export was cancelled"); @@ -3376,6 +4493,8 @@ export async function exportNativeStaticLayoutVideo( const nvidiaCudaSkipReason = await getExperimentalNvidiaCudaExportSkipReason(options); let shouldTryNvidiaCuda = nvidiaCudaSkipReason === null; + const requiresStrictHevcCuda = + options.videoCodec === "hevc" && options.encoderPreference === "hardware"; const validatedCudaFallbackCandidate = isValidatedNvidiaCudaFallbackCandidate(options); if ( @@ -3416,26 +4535,72 @@ export async function exportNativeStaticLayoutVideo( }, ); } - if (shouldTryNvidiaCuda && options.cursorTelemetry?.length) { - const cursorTelemetryPath = await prepareNvidiaCudaCursorTelemetry( - options, - path.join(chunkDirectory, "cursor-telemetry.json"), + if ( + !shouldTryNvidiaCuda && + ((options.overlayLayers?.length && + (options.zoomTelemetry?.length || options.cursorAtlasOwned === true)) || + options.temporalBlur) + ) { + throw new Error( + `CUDA composition is unavailable (${nvidiaCudaSkipReason ?? "unknown"}) while zoom motion blur${options.cursorAtlasOwned === true ? ", a native-owned cursor," : ""} and/or temporal zoom motion blur (${options.temporalBlur?.sampleCount ?? "n/a"} samples) are requested; the FFmpeg overlay route cannot preserve these effects${options.cursorAtlasOwned === true ? " and cannot draw the cursor the sidecar excluded" : ""}.`, ); - const cursorAtlas = await prepareNvidiaCudaCursorAtlas( - options, - path.join(chunkDirectory, "cursor-atlas-nvidia.png"), - path.join(chunkDirectory, "cursor-atlas-nvidia.tsv"), + } + if (options.cursorAtlasOwned === true && !options.cursorTelemetry?.length) { + throw new Error( + "Native cursor atlas ownership requires cursor telemetry; refusing to drop the cursor on a fallback route.", ); - shouldTryNvidiaCuda = Boolean(cursorTelemetryPath && cursorAtlas); - if (cursorTelemetryPath && cursorAtlas) { + } + if (shouldTryNvidiaCuda && options.cursorTelemetry?.length) { + if (options.overlayLayers?.length && options.cursorAtlasOwned !== true) { + // When overlay layers are present and the cursor is baked into the + // transparent sidecar, drawing it again natively would double-render + // and the atlas is intentionally absent. The Windows-GPU prep above + // left CSV telemetry/atlas paths on the options; strip them so the + // CUDA wrapper is never handed a CSV "cursor JSON" file (it + // JSON.parses the path and crashes) and never double-renders the + // baked cursor. experimentalNvidiaCudaOptions = { ...experimentalNvidiaCudaOptions, - cursorTelemetryPath, - cursorAtlasPath: cursorAtlas.atlasPath, - cursorAtlasMetadataPath: cursorAtlas.metadataPath, + ...resolveNvidiaCudaCursorAssets(experimentalNvidiaCudaOptions, true), }; + shouldTryNvidiaCuda = true; + } else { + // No overlay layers, or the renderer excluded cursor pixels from + // the overlay sidecar (cursorAtlasOwned): the CUDA compositor + // draws the native atlas cursor on top of the composed video. + const cursorTelemetryPath = await prepareNvidiaCudaCursorTelemetry( + options, + path.join(chunkDirectory, "cursor-telemetry.json"), + ); + const cursorAtlas = await prepareNvidiaCudaCursorAtlas( + options, + path.join(chunkDirectory, "cursor-atlas-nvidia.png"), + path.join(chunkDirectory, "cursor-atlas-nvidia.tsv"), + ); + if ( + options.cursorAtlasOwned === true && + (!cursorTelemetryPath || !cursorAtlas) + ) { + throw new Error( + "Native cursor atlas ownership could not prepare cursor telemetry/atlas assets; refusing to drop the cursor on the FFmpeg overlay route.", + ); + } + shouldTryNvidiaCuda = Boolean(cursorTelemetryPath && cursorAtlas); + if (cursorTelemetryPath && cursorAtlas) { + experimentalNvidiaCudaOptions = { + ...experimentalNvidiaCudaOptions, + cursorTelemetryPath, + cursorAtlasPath: cursorAtlas.atlasPath, + cursorAtlasMetadataPath: cursorAtlas.metadataPath, + }; + } } } + if (requiresStrictHevcCuda && !shouldTryNvidiaCuda) { + throw new Error( + `HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (${nvidiaCudaSkipReason ?? "cursor-atlas-unavailable"}) (noCpuFallback:true)`, + ); + } if (shouldTryNvidiaCuda) { try { @@ -3456,6 +4621,7 @@ export async function exportNativeStaticLayoutVideo( durationSec: options.durationSec, targetFrames: Math.ceil(options.durationSec * options.frameRate), requiresTimelineSync: shouldMuxAudioInline, + videoCodec, }, ); if (cudaValidationIssues.length > 0) { @@ -3470,6 +4636,20 @@ export async function exportNativeStaticLayoutVideo( ); } await validateRenderedVideoOutput(); + const overlaySidecarMetrics = + resolveNvidiaCudaOverlaySidecarSummaryMetrics(options); + if (overlaySidecarMetrics && cudaResult.summary.nativeSummary) { + // Additive renderer-derived overlay sidecar metrics ride on the + // parsed native summary so the completion log and chunk metrics + // surface them without changing the helper contract. + cudaResult.summary.nativeSummary = { + ...cudaResult.summary.nativeSummary, + ...overlaySidecarMetrics, + }; + } + const nativeSummaryMetrics = resolveNvidiaCudaNativeSummaryMetrics( + cudaResult.summary.nativeSummary, + ); console.info( "[native-static-layout-export] NVIDIA CUDA compositor completed", { @@ -3477,12 +4657,15 @@ export async function exportNativeStaticLayoutVideo( fps: cudaResult.summary.fps, targetFrames: cudaResult.summary.targetFrames, durationSec: cudaResult.summary.durationSec, + // timingsMs.nativeEncode is the full native helper-process wall + // time (spawn to exit: source decode, layout composition, NVENC, + // flush). It is NOT the NVENC-API encode time, which is + // reported separately as nativeSummary.nvencMs. + nativeEncodeWallMs: cudaResult.summary.timingsMs?.nativeEncode, nativeEncodeMs: cudaResult.summary.timingsMs?.nativeEncode, muxMs: cudaResult.summary.timingsMs?.mux, endToEndMs: cudaResult.summary.timingsMs?.endToEnd, - nativeFps: - cudaResult.summary.nativeSummary?.measuredFps ?? - cudaResult.summary.nativeSummary?.fps, + nativeFps: resolveNvidiaCudaNativeFps(cudaResult.summary), mappedDisplayFrames: cudaResult.summary.nativeSummary?.mappedDisplayFrames, selectedDisplayFrames: @@ -3499,6 +4682,17 @@ export async function exportNativeStaticLayoutVideo( cursorAtlas: cudaResult.summary.nativeSummary?.cursorAtlas, zoomOverlay: cudaResult.summary.nativeSummary?.zoomOverlay, zoomSamples: cudaResult.summary.nativeSummary?.zoomSamples, + overlayLayers: cudaResult.summary.nativeSummary?.overlayLayers, + ...nativeSummaryMetrics, + // The helper does not echo the configured temporal blur sample + // count in its summary yet; fall back to the renderer-resolved + // export plan so the log still exposes the requested count. + temporalBlurSampleCount: + nativeSummaryMetrics.temporalBlurSampleCount ?? + options.temporalBlur?.sampleCount, + metricInvariants: validateNvidiaCudaStageMetricInvariants( + cudaResult.summary, + ), }, ); metrics.chunkCount = 1; @@ -3519,16 +4713,37 @@ export async function exportNativeStaticLayoutVideo( if (session.terminating) { throw error; } + if (requiresStrictHevcCuda) { + throw new Error( + `HEVC Hardware NVIDIA CUDA compositor failed; refusing CPU, rawvideo, Breeze, or FFmpeg CUDA fallback (noCpuFallback:true): ${error instanceof Error ? error.message : String(error)}`, + ); + } metrics.fallbackChunkCount++; console.warn( "[native-static-layout-export] Experimental NVIDIA CUDA compositor failed or produced invalid output; falling back to Windows GPU compositor:", error, ); await removeTemporaryExportFile(videoOnlyPath); + if ( + (options.overlayLayers?.length && + (options.zoomTelemetry?.length || + options.cursorAtlasOwned === true)) || + options.temporalBlur + ) { + // Neither the D3D11 helper nor the FFmpeg overlay route can + // preserve spatial zoom blur over the transparent overlay + // sidecars, temporal zoom motion blur, or a native-owned cursor + // whose pixels the sidecar excluded; surface the failure so the + // renderer falls back to raw frames instead of silently dropping + // the effect. + throw new Error( + `CUDA composition failed while zoom motion blur${options.temporalBlur ? ` (temporal ${options.temporalBlur.sampleCount} samples)` : ""}${options.cursorAtlasOwned === true ? ", a native-owned cursor" : ""} and ${options.overlayLayers?.length ?? 0} overlay layer(s) are requested: ${error instanceof Error ? error.message : String(error)}`, + ); + } } } - if (!didRenderVideo) { + if (!didRenderVideo && videoCodec !== "hevc" && !options.overlayLayers?.length) { const gpuResult = await runExperimentalWindowsGpuStaticLayoutExport( experimentalGpuOptions, videoOnlyPath, @@ -3672,6 +4887,11 @@ export async function exportNativeStaticLayoutVideo( let fullBackend: NativeStaticLayoutBackend = "cuda-overlay"; let fallbackReason: string | undefined; if (!primaryResult.success) { + if (options.overlayLayers?.length) { + throw new Error( + `CUDA overlay-layer composition failed; refusing to drop ${options.overlayLayers.length} visual overlay layer(s): ${getFfmpegFailureMessage(primaryResult)}`, + ); + } fullBackend = "cuda-scale-cpu-pad"; fallbackReason = isNativeCudaOutOfMemory(primaryResult.stderr) ? "cuda-oom" @@ -3714,6 +4934,7 @@ export async function exportNativeStaticLayoutVideo( const baseConfig: NativeStaticLayoutExportArgsConfig = { inputPath: options.inputPath, outputPath, + videoCodec, width: options.width, height: options.height, frameRate: options.frameRate, @@ -3724,6 +4945,7 @@ export async function exportNativeStaticLayoutVideo( offsetX: options.offsetX, offsetY: options.offsetY, backgroundColor: options.backgroundColor, + videoEncoder: nativeVideoEncoder, startSec: chunk.startSec, durationSec: chunk.durationSec, }; @@ -3798,6 +5020,10 @@ export async function exportNativeStaticLayoutVideo( return { outputPath: videoOnlyPath, metrics, + videoCodec, + encoderPreference, + encoderName: "nvidia-cuda-compositor", + route: "nvidia-cuda-compositor", }; } const audioMuxProgressStart = 97.25; @@ -3827,9 +5053,17 @@ export async function exportNativeStaticLayoutVideo( ); Object.assign(metrics, finalized.metrics); outputPathToKeep = finalized.outputPath; + const route = metrics.chunks[0]?.backend; + if (!route) { + throw new Error("Native static-layout export did not report a route"); + } return { outputPath: finalized.outputPath, metrics, + videoCodec, + encoderPreference, + encoderName: nativeVideoEncoder, + route, }; } catch (error) { await removeTemporaryExportFile(videoOnlyPath); @@ -3899,8 +5133,8 @@ export async function probeNativeVideoEncoder( const args = buildNativeVideoExportArgs( encoderName, { - width: 64, - height: 64, + width: NATIVE_ENCODER_PROBE_DIMENSION, + height: NATIVE_ENCODER_PROBE_DIMENSION, frameRate: 1, bitrate: 1_500_000, encodingMode, @@ -3938,38 +5172,66 @@ export async function probeNativeVideoEncoder( resolve(code === 0); }); - process.stdin.end(Buffer.alloc(getNativeVideoInputByteSize(64, 64), 0)); + process.stdin.end( + Buffer.alloc( + getNativeVideoInputByteSize( + NATIVE_ENCODER_PROBE_DIMENSION, + NATIVE_ENCODER_PROBE_DIMENSION, + ), + 0, + ), + ); }); } export async function resolveNativeVideoEncoder( ffmpegPath: string, encodingMode: NativeExportEncodingMode, + codec: "h264" | "hevc" = "h264", + preference: "auto" | "hardware" | "cpu" = "auto", ) { if ( cachedNativeVideoEncoder?.ffmpegPath === ffmpegPath && - cachedNativeVideoEncoder?.encodingMode === encodingMode + cachedNativeVideoEncoder?.encodingMode === encodingMode && + cachedNativeVideoEncoder?.codec === codec && + cachedNativeVideoEncoder?.preference === preference ) { return cachedNativeVideoEncoder.encoderName; } const availableEncoders = await getAvailableNativeVideoEncoders(ffmpegPath); - const candidates = [ - ...new Set([...getPreferredNativeVideoEncoders(process.platform), "libx264"]), - ]; + const candidates = getNativeEncoderCandidates(codec, preference, process.platform); + const usableCandidates = candidates.filter((encoderName) => availableEncoders.has(encoderName)); - for (const encoderName of candidates) { - if (!availableEncoders.has(encoderName)) { - continue; - } + if (usableCandidates.length === 0) { + throw new Error( + `No usable FFmpeg ${codec.toUpperCase()} encoder was available for native export (preference: ${preference})`, + ); + } + for (const encoderName of usableCandidates) { if (await probeNativeVideoEncoder(ffmpegPath, encoderName, encodingMode)) { - setCachedNativeVideoEncoder({ ffmpegPath, encodingMode, encoderName }); + setCachedNativeVideoEncoder({ + ffmpegPath, + encodingMode, + codec, + preference, + encoderName, + }); return encoderName; } } - throw new Error("No usable FFmpeg encoder was available for native export"); + if (preference === "hardware") { + throw new Error( + `No usable hardware FFmpeg ${codec.toUpperCase()} encoder was available for native export. ` + + `Tried: ${usableCandidates.join(", ")}. Install a supported hardware encoder or choose CPU/auto.`, + ); + } + + throw new Error( + `No usable FFmpeg ${codec.toUpperCase()} encoder was available for native export (preference: ${preference})`, + ); } export function canCopyAudioCodecIntoMp4(codec?: string | null) { diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts index 36bcaf064..16413c055 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts @@ -46,7 +46,7 @@ describe("planNativeStaticLayoutRoutes", () => { d3d11: d3d11Probe, source, }), - ).toEqual({ + ).toMatchObject({ selectedRoute: "nvidia-cuda-compositor", decisions: [ { @@ -124,7 +124,7 @@ describe("planNativeStaticLayoutRoutes", () => { d3d11, source, }), - ).toEqual({ + ).toMatchObject({ selectedRoute: "ffmpeg-static-layout", decisions: [ { @@ -149,6 +149,77 @@ describe("planNativeStaticLayoutRoutes", () => { }); }); + it("routes HEVC only through NVIDIA CUDA", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.videoCodec).toBe("hevc"); + expect(plan.encoderPreference).toBe("auto"); + expect(plan.decisions).toEqual( + expect.arrayContaining([ + { + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + { + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + ]), + ); + }); + + it("falls back to rawvideo when HEVC CUDA is unavailable", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "hardware", + cuda: { ...cudaProbe, skipReason: "nvidia-gpu-unavailable" }, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.fallbackReason).toBe("hevc-hardware-route-unavailable:nvidia-gpu-unavailable"); + expect(plan.decisions).toEqual( + expect.arrayContaining([ + { + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + { + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }, + ]), + ); + }); + + it("keeps CPU preference out of every GPU route", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "h264", + encoderPreference: "cpu", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.fallbackReason).toBe("encoder-preference-cpu-requires-native-rawvideo"); + expect(plan.decisions.every((decision) => decision.status === "rejected")).toBe(true); + }); + it("preserves proxy source metadata for route diagnostics", () => { const proxiedSource = { inputCodec: "vp9", diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts index 3ee6c4e73..92706e6b2 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts @@ -1,9 +1,14 @@ -import type { NativeVideoExportAudioMode } from "../nativeVideoExport"; +import type { + ExportEncoderPreference, + ExportVideoCodec, + NativeVideoExportAudioMode, +} from "../nativeVideoExport"; export type NativeStaticLayoutRoute = | "nvidia-cuda-compositor" | "windows-d3d11-compositor" | "ffmpeg-static-layout"; +export type NativeStaticLayoutFallbackRoute = "native-rawvideo"; export interface NativeStaticLayoutRouteDecision { route: NativeStaticLayoutRoute; @@ -44,21 +49,140 @@ export interface NativeStaticLayoutRouteSource { } export interface NativeStaticLayoutRoutePlan { - selectedRoute: NativeStaticLayoutRoute; + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + selectedRoute: NativeStaticLayoutRoute | null; + fallbackRoute: NativeStaticLayoutFallbackRoute | null; + fallbackReason: string | null; decisions: NativeStaticLayoutRouteDecision[]; cuda: NvidiaCudaExportCapabilityProbe; d3d11: WindowsD3D11ExportCapabilityProbe; source: NativeStaticLayoutRouteSource; } +function createRawVideoFallbackPlan(options: { + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + cuda: NvidiaCudaExportCapabilityProbe; + d3d11: WindowsD3D11ExportCapabilityProbe; + source: NativeStaticLayoutRouteSource; + reason: string; + cudaReason: string; +}) { + const { videoCodec, encoderPreference, cuda, d3d11, source } = options; + return { + videoCodec, + encoderPreference, + selectedRoute: null, + fallbackRoute: "native-rawvideo" as const, + fallbackReason: options.reason, + decisions: [ + { + route: "nvidia-cuda-compositor" as const, + status: "rejected" as const, + reasons: [options.cudaReason], + }, + { + route: "windows-d3d11-compositor" as const, + status: "rejected" as const, + reasons: [ + videoCodec === "hevc" ? "hevc-requires-nvidia-cuda-compositor" : options.reason, + ], + }, + { + route: "ffmpeg-static-layout" as const, + status: "rejected" as const, + reasons: [ + videoCodec === "hevc" ? "hevc-requires-nvidia-cuda-compositor" : options.reason, + ], + }, + ], + cuda, + d3d11, + source, + } satisfies NativeStaticLayoutRoutePlan; +} + export function planNativeStaticLayoutRoutes(options: { + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; cuda: NvidiaCudaExportCapabilityProbe; d3d11: WindowsD3D11ExportCapabilityProbe; source: NativeStaticLayoutRouteSource; }): NativeStaticLayoutRoutePlan { + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; const { cuda, d3d11, source } = options; const decisions: NativeStaticLayoutRouteDecision[] = []; + if (encoderPreference === "cpu") { + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: "encoder-preference-cpu-requires-native-rawvideo", + cudaReason: "encoder-preference-cpu-never-enters-gpu-compositor", + }); + } + + if (videoCodec === "hevc") { + if (!cuda.skipReason) { + decisions.push({ + route: "nvidia-cuda-compositor", + status: "selected", + reasons: ["cuda-wrapper-and-nvidia-gpu-available-for-hevc"], + }); + decisions.push({ + route: "windows-d3d11-compositor", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }); + decisions.push({ + route: "ffmpeg-static-layout", + status: "rejected", + reasons: ["hevc-requires-nvidia-cuda-compositor"], + }); + return { + videoCodec, + encoderPreference, + selectedRoute: "nvidia-cuda-compositor", + fallbackRoute: null, + fallbackReason: null, + decisions, + cuda, + d3d11, + source, + }; + } + + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: + encoderPreference === "hardware" + ? `hevc-hardware-route-unavailable:${cuda.skipReason}` + : `hevc-cuda-unavailable:${cuda.skipReason}`, + cudaReason: cuda.skipReason, + }); + } + + if (encoderPreference === "hardware") { + return createRawVideoFallbackPlan({ + videoCodec, + encoderPreference, + cuda, + d3d11, + source, + reason: "encoder-preference-hardware-requires-native-rawvideo", + cudaReason: "explicit-hardware-preference-requires-native-rawvideo", + }); + } + if (!cuda.skipReason) { decisions.push({ route: "nvidia-cuda-compositor", @@ -78,7 +202,11 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["native-gpu-runtime-fallback"], }); return { + videoCodec, + encoderPreference, selectedRoute: "nvidia-cuda-compositor", + fallbackRoute: null, + fallbackReason: null, decisions, cuda, d3d11, @@ -104,7 +232,11 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["windows-d3d11-runtime-fallback"], }); return { + videoCodec, + encoderPreference, selectedRoute: "windows-d3d11-compositor", + fallbackRoute: null, + fallbackReason: null, decisions, cuda, d3d11, @@ -123,7 +255,11 @@ export function planNativeStaticLayoutRoutes(options: { reasons: ["native-gpu-routes-unavailable"], }); return { + videoCodec, + encoderPreference, selectedRoute: "ffmpeg-static-layout", + fallbackRoute: null, + fallbackReason: null, decisions, cuda, d3d11, diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index 5992cadf3..d5e26f216 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { captureCanvasFrameForNativeExport } from "../../src/lib/exporter/nativeFrameCapture"; import { ATEMPO_FILTER_EPSILON } from "./ffmpeg/filters"; import { buildEditedTrackSourceAudioFilter, @@ -8,11 +9,85 @@ import { buildNativePrecompositedStaticLayoutArgs, buildNativeStaticBackgroundRenderArgs, buildNativeStaticLayoutChunks, + buildNativeVideoExportArgs, buildTrimmedSourceAudioFilter, createNativeSquircleMaskPgmBuffer, + getCpuEncoderForCodec, + getNativeEncoderCandidates, isNativeCudaOutOfMemory, } from "./nativeVideoExport"; +const childProcessMocks = vi.hoisted(() => ({ + encoderNames: "", + failingEncoders: new Set(), + execFile: vi.fn( + ( + _file: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, result?: { stdout: string; stderr: string }) => void, + ) => { + if (Array.isArray(args) && args.includes("-encoders")) { + cb(null, { stdout: childProcessMocks.encoderNames, stderr: "" }); + } else { + cb(null, { stdout: "", stderr: "" }); + } + }, + ), + spawn: vi.fn((_file: string, args: string[]) => { + let encoder = ""; + const codecIndex = args.indexOf("-c:v"); + if (codecIndex >= 0) { + encoder = args[codecIndex + 1]; + } + const exitCode = childProcessMocks.failingEncoders.has(encoder) ? 1 : 0; + return { + stdin: { + end: vi.fn(() => undefined), + destroy: vi.fn(), + destroyed: false, + writableEnded: false, + }, + stderr: { on: vi.fn() }, + on: vi.fn((event: string, cb: (code: number) => void) => { + if (event === "close") { + setTimeout(() => cb(exitCode), 0); + } + }), + kill: vi.fn(), + }; + }), +})); + +vi.mock("electron", () => ({ + app: { + getAppPath: vi.fn(() => process.cwd()), + getGPUInfo: vi.fn(async () => ({ gpuDevice: [] })), + getPath: vi.fn(() => process.env.TEMP ?? process.cwd()), + isPackaged: false, + }, + powerSaveBlocker: { + start: vi.fn(() => 1), + isStarted: vi.fn(() => false), + stop: vi.fn(), + }, +})); + +vi.mock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: vi.fn(() => "ffmpeg"), + getFfprobeBinaryPath: vi.fn(() => "ffprobe"), +})); + +vi.mock("node:child_process", () => ({ + execFile: childProcessMocks.execFile, + spawn: childProcessMocks.spawn, +})); + +import { resolveNativeVideoEncoder } from "./export/native-video"; +// Use the real in-memory state module so we can seed/read the native encoder +// cache through its real setter for the cache-identity tests below. +import { setCachedNativeVideoEncoder } from "./state"; + describe("buildTrimmedSourceAudioFilter", () => { it("concatenates trimmed source segments into a single output label", () => { expect( @@ -160,6 +235,62 @@ describe("native static layout command builders", () => { expect(args).toEqual(expect.arrayContaining(["-ss", "120.000", "-t", "60.000"])); }); + it("selects HEVC NVENC without changing the CUDA filtergraph", () => { + const args = buildNativeCudaOverlayStaticLayoutArgs({ + ...baseConfig, + videoCodec: "hevc", + }); + + expect(args).toContain("hevc_nvenc"); + expect(args).not.toContain("h264_nvenc"); + expect(args).toEqual( + expect.arrayContaining([expect.stringContaining("overlay_cuda=192:108")]), + ); + }); + + it("adds sorted transparent RGBA overlay inputs to the CUDA filtergraph", () => { + const args = buildNativeCudaOverlayStaticLayoutArgs({ + ...baseConfig, + overlayLayers: [ + { + id: "caption", + order: 2, + path: "caption.rgba", + x: 0, + y: 800, + width: 1920, + height: 280, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "cursor", + order: 1, + path: "cursor.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filter = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual( + expect.arrayContaining(["-f", "rawvideo", "cursor.rgba", "caption.rgba"]), + ); + expect(filter).toContain("[1:v]format=rgba[overlay_0]"); + expect(filter).toContain("overlay=0:0"); + expect(filter).toContain("[2:v]format=rgba[overlay_1]"); + expect(filter).toContain("overlay=0:800"); + }); + it("builds the stable CUDA scale plus CPU pad fallback command", () => { const args = buildNativeCudaScaleCpuPadStaticLayoutArgs(baseConfig); @@ -248,6 +379,33 @@ describe("native static layout command builders", () => { expect(pixels[4 * 8 + 4]).toBe(255); }); + it("preserves overlay layers in the precomposited CPU composition branch", () => { + const args = buildNativePrecompositedStaticLayoutArgs({ + ...baseConfig, + staticBackgroundPath: "background.png", + overlayLayers: [ + { + id: "effects", + order: 0, + path: "effects.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual(expect.arrayContaining(["-f", "rawvideo", "effects.rgba"])); + expect(filterComplex).toContain("[2:v]format=rgba[overlay_0]"); + expect(filterComplex).toContain("[layout][overlay_0]overlay=x=0:y=0:format=auto"); + }); + it("splits long exports into bounded chunks", () => { expect(buildNativeStaticLayoutChunks(367.5, 120)).toEqual([ { index: 0, startSec: 0, durationSec: 120 }, @@ -286,3 +444,234 @@ describe("native static layout command builders", () => { expect(isNativeCudaOutOfMemory("FFmpeg exited with code 1")).toBe(false); }); }); + +describe("getNativeEncoderCandidates", () => { + it("orders H.265 hardware candidates highest for Auto on Windows", () => { + expect(getNativeEncoderCandidates("hevc", "auto", "win32")).toEqual([ + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + "libx265", + ]); + }); + + it("returns only hardware candidates for the hardware preference", () => { + expect(getNativeEncoderCandidates("hevc", "hardware", "win32")).toEqual([ + "hevc_nvenc", + "hevc_qsv", + "hevc_amf", + "hevc_mf", + ]); + expect(getNativeEncoderCandidates("hevc", "hardware", "darwin")).toEqual([ + "hevc_videotoolbox", + ]); + expect(getNativeEncoderCandidates("h264", "hardware", "linux")).toEqual([ + "h264_nvenc", + "h264_qsv", + ]); + }); + + it("returns only the CPU encoder for the cpu preference", () => { + expect(getNativeEncoderCandidates("hevc", "cpu", "linux")).toEqual(["libx265"]); + expect(getNativeEncoderCandidates("h264", "cpu", "win32")).toEqual(["libx264"]); + }); + + it("orders H.264 Linux hardware before the CPU fallback for Auto", () => { + expect(getNativeEncoderCandidates("h264", "auto", "linux")).toEqual([ + "h264_nvenc", + "h264_qsv", + "libx264", + ]); + }); + + it("maps each codec to its CPU encoder", () => { + expect(getCpuEncoderForCodec("h264")).toBe("libx264"); + expect(getCpuEncoderForCodec("hevc")).toBe("libx265"); + }); +}); + +describe("native raw-frame orientation", () => { + it("keeps synthetic top and bottom rows top-down without an FFmpeg flip", async () => { + class SyntheticVideoFrame { + async copyTo(destination: Uint8Array): Promise { + destination.set([255, 0, 0, 255, 0, 0, 255, 255]); + } + + close(): void {} + } + + vi.stubGlobal("VideoFrame", SyntheticVideoFrame); + const frame = await captureCanvasFrameForNativeExport( + { width: 1, height: 2 } as HTMLCanvasElement, + 0, + true, + ); + expect([...frame]).toEqual([255, 0, 0, 255, 0, 0, 255, 255]); + + const args = buildNativeVideoExportArgs( + "libx264", + { + width: 1, + height: 2, + frameRate: 30, + bitrate: 1_500_000, + encodingMode: "fast", + inputMode: "rawvideo", + }, + "out.mp4", + ); + expect(args).toEqual(expect.arrayContaining(["-f", "rawvideo", "-pix_fmt", "rgba"])); + expect(args).not.toContain("-vf"); + expect(args).not.toContain("vflip"); + }); +}); + +describe("buildNativeVideoExportArgs codec-aware", () => { + const base = { + width: 1920, + height: 1080, + frameRate: 60, + bitrate: 8_000_000, + encodingMode: "quality" as const, + }; + + it("libx265 includes bitrate, GOP, pixel format, and MP4 flags", () => { + const args = buildNativeVideoExportArgs( + "libx265", + { ...base, videoCodec: "hevc", encoderPreference: "cpu" }, + "out.mp4", + ); + expect(args[args.indexOf("-c:v") + 1]).toBe("libx265"); + expect(args[args.indexOf("-preset") + 1]).toBe("slow"); + expect(args[args.indexOf("-g") + 1]).toBe("300"); + expect(args).toContain("-b:v"); + expect(args[args.lastIndexOf("-pix_fmt") + 1]).toBe("yuv420p"); + expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart"); + expect(args).toContain("out.mp4"); + }); + + it("hevc_nvenc includes bitrate, GOP, pixel format, and MP4 flags", () => { + const args = buildNativeVideoExportArgs( + "hevc_nvenc", + { ...base, videoCodec: "hevc", encoderPreference: "hardware" }, + "out.mp4", + ); + expect(args[args.indexOf("-c:v") + 1]).toBe("hevc_nvenc"); + expect(args).toContain("-preset"); + expect(args[args.indexOf("-g") + 1]).toBe("300"); + expect(args).toContain("-b:v"); + expect(args[args.lastIndexOf("-pix_fmt") + 1]).toBe("yuv420p"); + expect(args[args.indexOf("-movflags") + 1]).toBe("+faststart"); + expect(args).toContain("out.mp4"); + }); + + it("libx264 CPU keeps its existing preset tuning", () => { + const args = buildNativeVideoExportArgs( + "libx264", + { ...base, videoCodec: "h264", encoderPreference: "cpu" }, + "out.mp4", + ); + expect(args[args.indexOf("-preset") + 1]).toBe("slow"); + // quality mode => -preset slow on libx264 (no -tune for quality). + expect(args).toContain("-preset"); + }); +}); + +describe("resolveNativeVideoEncoder", () => { + const encoderListing = (...names: string[]) => + names.map((name) => ` V....D ${name} ffmpeg-${name} encoder\r\n`).join(""); + + beforeEach(() => { + childProcessMocks.execFile.mockClear(); + childProcessMocks.spawn.mockClear(); + childProcessMocks.failingEncoders = new Set(); + setCachedNativeVideoEncoder(null); + childProcessMocks.encoderNames = encoderListing("libx264", "libx265"); + }); + + it("resolves H.264 CPU to libx264", async () => { + childProcessMocks.encoderNames = encoderListing("libx264"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "cpu")).resolves.toBe( + "libx264", + ); + const probeArgs = childProcessMocks.spawn.mock.calls[0]?.[1] as string[]; + expect(probeArgs).toContain("libx264"); + }); + + it("resolves HEVC CPU to libx265", async () => { + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "cpu")).resolves.toBe( + "libx265", + ); + const probeArgs = childProcessMocks.spawn.mock.calls[0]?.[1] as string[]; + expect(probeArgs).toContain("libx265"); + }); + + it("auto falls back from a failing hardware encoder to CPU", async () => { + const hardware = getNativeEncoderCandidates("hevc", "hardware", process.platform); + childProcessMocks.encoderNames = encoderListing(...hardware, "libx265"); + childProcessMocks.failingEncoders = new Set(hardware); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "auto")).resolves.toBe( + "libx265", + ); + }); + + it("hardware-only does not silently fall back to CPU", async () => { + const hardware = getNativeEncoderCandidates("hevc", "hardware", process.platform); + childProcessMocks.encoderNames = encoderListing(...hardware, "libx265"); + childProcessMocks.failingEncoders = new Set(hardware); + await expect( + resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "hardware"), + ).rejects.toThrow(/hardware/i); + const probedEncoders = ( + childProcessMocks.spawn.mock.calls as Array<[string, string[]]> + ).map(([, args]) => args[args.indexOf("-c:v") + 1]); + expect(probedEncoders).not.toContain("libx265"); + }); + + it("errors when no available hardware encoder can be probed", async () => { + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect( + resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "hardware"), + ).rejects.toThrow(/hardware/i); + expect(childProcessMocks.spawn).not.toHaveBeenCalled(); + }); + + it("includes codec and preference in the cache identity", async () => { + // Seed the cache with an H.264/auto entry via the real state setter. + setCachedNativeVideoEncoder({ + ffmpegPath: "ffmpeg", + encodingMode: "balanced", + codec: "h264", + preference: "auto", + encoderName: "libx264", + }); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "auto")).resolves.toBe( + "libx264", + ); + expect(childProcessMocks.execFile).not.toHaveBeenCalled(); + expect(childProcessMocks.spawn).not.toHaveBeenCalled(); + + // A different codec must not reuse the cached H.264/auto entry. + setCachedNativeVideoEncoder(null); + childProcessMocks.encoderNames = encoderListing("libx265"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "hevc", "auto")).resolves.toBe( + "libx265", + ); + + // A different preference must not collide with the cached H.264/auto entry. + setCachedNativeVideoEncoder({ + ffmpegPath: "ffmpeg", + encodingMode: "balanced", + codec: "h264", + preference: "auto", + encoderName: "libx264", + }); + childProcessMocks.encoderNames = encoderListing("libx264"); + await expect(resolveNativeVideoEncoder("ffmpeg", "balanced", "h264", "cpu")).resolves.toBe( + "libx264", + ); + expect(childProcessMocks.execFile).toHaveBeenCalled(); + }); +}); diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index 07aa64a91..daa549c0b 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -1,8 +1,12 @@ +import type { NativeStaticLayoutOverlayLayer } from "../../src/lib/exporter/nativeStaticLayoutOverlays"; import { getShadowFilterPadding, VIDEO_SHADOW_LAYER_PROFILES, } from "../../src/lib/exporter/shadowProfile"; +import type { ExportEncoderPreference, ExportVideoCodec } from "../../src/lib/exporter/types"; import { getSquirclePathPoints } from "../../src/lib/geometry/squircle"; +export type { ExportEncoderPreference, ExportVideoCodec }; + import { ATEMPO_FILTER_EPSILON, buildAtempoFilters } from "./ffmpeg/filters"; const NATIVE_EXPORT_INPUT_BYTES_PER_PIXEL = 4; @@ -23,6 +27,8 @@ export interface NativeVideoExportStartOptions { bitrate: number; encodingMode: NativeExportEncodingMode; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; } export interface NativeVideoExportAudioSegment { @@ -67,6 +73,7 @@ export type NativeStaticLayoutBackend = export interface NativeStaticLayoutExportArgsConfig { inputPath: string; outputPath: string; + videoCodec?: ExportVideoCodec; width: number; height: number; frameRate: number; @@ -89,6 +96,12 @@ export interface NativeStaticLayoutExportArgsConfig { shadowIntensity?: number; startSec?: number; durationSec?: number; + overlayLayers?: NativeStaticLayoutOverlayLayer[]; + videoEncoder?: string; +} + +function getNativeStaticLayoutVideoEncoder(codec: ExportVideoCodec = "h264") { + return codec === "hevc" ? "hevc_nvenc" : "h264_nvenc"; } export interface NativeStaticLayoutChunk { @@ -114,17 +127,58 @@ export function parseAvailableFfmpegEncoders(stdout: string): Set { return encoders; } -export function getPreferredNativeVideoEncoders(platform: NodeJS.Platform): string[] { +function getHardwareEncoderCandidates( + codec: ExportVideoCodec, + platform: NodeJS.Platform, +): string[] { + if (codec === "hevc") { + switch (platform) { + case "darwin": + return ["hevc_videotoolbox"]; + case "win32": + return ["hevc_nvenc", "hevc_qsv", "hevc_amf", "hevc_mf"]; + case "linux": + return ["hevc_nvenc", "hevc_qsv"]; + default: + return []; + } + } + switch (platform) { case "darwin": - return ["h264_videotoolbox", "libx264"]; + return ["h264_videotoolbox"]; case "win32": - return ["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf", "libx264"]; + return ["h264_nvenc", "h264_qsv", "h264_amf", "h264_mf"]; case "linux": - return ["h264_nvenc", "h264_qsv", "libx264"]; + return ["h264_nvenc", "h264_qsv"]; default: - return ["libx264"]; + return []; + } +} + +export function getCpuEncoderForCodec(codec: ExportVideoCodec): string { + return codec === "hevc" ? "libx265" : "libx264"; +} + +export function getNativeEncoderCandidates( + codec: ExportVideoCodec, + preference: ExportEncoderPreference, + platform: NodeJS.Platform, +): string[] { + const hardware = getHardwareEncoderCandidates(codec, platform); + const cpu = [getCpuEncoderForCodec(codec)]; + + if (preference === "cpu") { + return cpu; + } + if (preference === "hardware") { + return hardware; } + return [...hardware, ...cpu]; +} + +export function getPreferredNativeVideoEncoders(platform: NodeJS.Platform): string[] { + return getNativeEncoderCandidates("h264", "auto", platform); } function getLibx264ModeArgs(encodingMode: NativeExportEncodingMode): string[] { @@ -139,6 +193,18 @@ function getLibx264ModeArgs(encodingMode: NativeExportEncodingMode): string[] { } } +function getLibx265ModeArgs(encodingMode: NativeExportEncodingMode): string[] { + switch (encodingMode) { + case "fast": + return ["-preset", "ultrafast"]; + case "quality": + return ["-preset", "slow"]; + case "balanced": + default: + return ["-preset", "medium"]; + } +} + function getBitrateArgs(bitrate: number): string[] { const effectiveBitrate = Math.max(1_500_000, Math.round(bitrate)); const maxRate = Math.max(effectiveBitrate, Math.round(effectiveBitrate * 1.2)); @@ -295,8 +361,6 @@ export function buildNativeVideoExportArgs( String(options.frameRate), "-i", "pipe:0", - "-vf", - "vflip", "-an", "-c:v", encoder, @@ -307,6 +371,10 @@ export function buildNativeVideoExportArgs( if (encoder === "libx264") { args.push(...getLibx264ModeArgs(options.encodingMode)); + } else if (encoder === "libx265") { + args.push(...getLibx265ModeArgs(options.encodingMode)); + } else if (encoder === "hevc_nvenc") { + args.push(...getNvencStaticLayoutModeArgs(options.encodingMode)); } args.push("-pix_fmt", "yuv420p", "-movflags", "+faststart", outputPath); @@ -319,23 +387,63 @@ export function buildNativeCudaOverlayStaticLayoutArgs( const backgroundColor = formatFfmpegColor(config.backgroundColor); const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); const args = ["-y", "-hide_banner", "-loglevel", "error"]; + const overlayLayers = [...(config.overlayLayers ?? [])].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); + args.push("-hwaccel", "cuda", "-hwaccel_output_format", "cuda", "-i", config.inputPath); + for (const layer of overlayLayers) { + args.push( + "-f", + "rawvideo", + "-pix_fmt", + "rgba", + "-s:v", + `${layer.width}x${layer.height}`, + "-framerate", + String(layer.frameRate), + "-i", + layer.path, + ); + } + + let filterComplex = + `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];` + + `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];` + + `[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`; + if (overlayLayers.length > 0) { + const filterParts = [ + `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=rgba[bg]`, + `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fg]`, + `[bg][fg]overlay=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat[layout]`, + ]; + let currentLabel = "layout"; + for (const [index, layer] of overlayLayers.entries()) { + const inputIndex = index + 1; + const overlayLabel = `overlay_${index}`; + const nextLabel = index === overlayLayers.length - 1 ? "out" : `layout_${index}`; + filterParts.push( + `[${inputIndex}:v]format=rgba[${overlayLabel}]`, + `[${currentLabel}][${overlayLabel}]overlay=${layer.x}:${layer.y}:shortest=0:repeatlast=1:eof_action=repeat[${nextLabel}]`, + ); + currentLabel = nextLabel; + } + filterParts.push( + `[${currentLabel}]trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`, + ); + filterComplex = filterParts.join(";"); + } + args.push( - "-hwaccel", - "cuda", - "-hwaccel_output_format", - "cuda", - "-i", - config.inputPath, "-filter_complex", - `color=c=${backgroundColor}:s=${config.width}x${config.height}:r=${config.frameRate}:d=${durationSec},format=nv12,hwupload_cuda[bg];[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12,fps=${config.frameRate}[fg];[bg][fg]overlay_cuda=${config.offsetX}:${config.offsetY}:shortest=0:repeatlast=1:eof_action=repeat,trim=duration=${durationSec},setpts=PTS-STARTPTS[out]`, + filterComplex, "-map", "[out]", "-an", "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-movflags", @@ -348,6 +456,9 @@ export function buildNativeCudaOverlayStaticLayoutArgs( export function buildNativeCudaScaleCpuPadStaticLayoutArgs( config: NativeStaticLayoutExportArgsConfig, ): string[] { + if (config.overlayLayers?.length) { + throw new Error("CUDA scale/pad fallback cannot preserve native overlay layers"); + } const backgroundColor = formatFfmpegColor(config.backgroundColor); const args = ["-y", "-hide_banner", "-loglevel", "error"]; pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); @@ -366,7 +477,7 @@ export function buildNativeCudaScaleCpuPadStaticLayoutArgs( "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-pix_fmt", @@ -513,11 +624,43 @@ export function buildNativePrecompositedStaticLayoutArgs( config.maskPath, ); } + for (const layer of config.overlayLayers ?? []) { + args.push( + "-f", + "rawvideo", + "-pix_fmt", + "rgba", + "-s:v", + `${layer.width}x${layer.height}`, + "-framerate", + String(layer.frameRate), + "-i", + layer.path, + ); + } const foregroundFilter = `[0:v]scale_cuda=w=${config.contentWidth}:h=${config.contentHeight}:format=nv12:passthrough=0,hwdownload,format=nv12,fps=${config.frameRate},format=rgba[fgbase]`; const maskFilter = useMask ? ";[2:v]format=gray[mask];[fgbase][mask]alphamerge[fg]" : ""; const foregroundLabel = useMask ? "fg" : "fgbase"; - const filterComplex = `${foregroundFilter}${maskFilter};[1:v]format=rgba[bg];[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto,trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`; + const filterParts = [ + `${foregroundFilter}${maskFilter}`, + `[1:v]format=rgba[bg]`, + `[bg][${foregroundLabel}]overlay=x=${config.offsetX}:y=${config.offsetY}:format=auto[layout]`, + ]; + let currentLabel = "layout"; + const firstOverlayInputIndex = useMask ? 3 : 2; + for (const [index, layer] of (config.overlayLayers ?? []).entries()) { + const nextLabel = `layout_overlay_${index}`; + filterParts.push( + `[${firstOverlayInputIndex + index}:v]format=rgba[overlay_${index}]`, + `[${currentLabel}][overlay_${index}]overlay=x=${layer.x}:y=${layer.y}:format=auto[${nextLabel}]`, + ); + currentLabel = nextLabel; + } + filterParts.push( + `[${currentLabel}]trim=duration=${durationSec},setpts=PTS-STARTPTS,format=yuv420p[out]`, + ); + const filterComplex = filterParts.join(";"); args.push( "-filter_complex", @@ -528,7 +671,7 @@ export function buildNativePrecompositedStaticLayoutArgs( "-r", String(config.frameRate), "-c:v", - "h264_nvenc", + config.videoEncoder ?? getNativeStaticLayoutVideoEncoder(config.videoCodec), ...getNvencStaticLayoutModeArgs(config.encodingMode), ...getBitrateArgs(config.bitrate), "-pix_fmt", diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index c4410a271..5634cab8c 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -5,12 +5,6 @@ import path from "node:path"; import type { Readable, Writable } from "node:stream"; import type { SaveDialogOptions } from "electron"; import { app, BrowserWindow, dialog, ipcMain } from "electron"; -import { - parseCaptionSidecarPayload, - type CaptionSidecarPayload, - withCaptionSidecarMessage, - writeCaptionSidecarsBestEffort, -} from "./exportCaptionSidecars"; import { closeExportStream, isOwnedExportPath, @@ -20,9 +14,12 @@ import { writeToExportStream, } from "../export/exportStream"; import { + attachNativeVideoExportFramePort, + closeNativeVideoExportFramePort, enqueueNativeVideoExportFrameWrite, enqueueNativeVideoExportFrameWrites, exportNativeStaticLayoutVideo, + flushNativeVideoExportFramePortPendingRequests, flushNativeVideoExportPendingWriteRequests, getNativeExportCapabilities, getNativeVideoExportMaxQueuedWriteBytes, @@ -38,6 +35,7 @@ import { probeNativeVideoMetadata, removeTemporaryExportFile, resolveNativeVideoEncoder, + sendNativeVideoExportFramePortError, sendNativeVideoExportWriteFrameResult, settleNativeVideoExportWriteFrameRequest, } from "../export/native-video"; @@ -45,12 +43,20 @@ import { getFfmpegBinaryPath } from "../ffmpeg/binary"; import { buildNativeH264StreamExportArgs, buildNativeVideoExportArgs, + type ExportEncoderPreference, + type ExportVideoCodec, getNativeVideoInputByteSize, type NativeExportEncodingMode, type NativeVideoExportFinishOptions, } from "../nativeVideoExport"; import { isAllowedLocalReadPath, resolveApprovedLocalMediaPath } from "../project/manager"; import { approveUserPath } from "../utils"; +import { + type CaptionSidecarPayload, + parseCaptionSidecarPayload, + withCaptionSidecarMessage, + writeCaptionSidecarsBestEffort, +} from "./exportCaptionSidecars"; function getPartialExportDestinationPath(destinationPath: string) { const parsed = path.parse(destinationPath); @@ -75,12 +81,7 @@ export async function moveExportedTempFile(tempPath: string, destinationPath: st return; } catch (error) { const code = (error as NodeJS.ErrnoException).code; - if ( - code !== "EXDEV" && - code !== "EPERM" && - code !== "ENOTEMPTY" && - code !== "EEXIST" - ) { + if (code !== "EXDEV" && code !== "EPERM" && code !== "ENOTEMPTY" && code !== "EEXIST") { throw error; } // Cross-device or Windows permission quirks — fall back to copy + unlink so @@ -113,9 +114,7 @@ export async function moveExportedTempFile(tempPath: string, destinationPath: st await fs.rename(partialDestinationPath, destinationPath); } catch (replaceError) { if (movedExistingDestination) { - await fs - .rename(backupDestinationPath, destinationPath) - .catch(() => undefined); + await fs.rename(backupDestinationPath, destinationPath).catch(() => undefined); } throw replaceError; } @@ -194,6 +193,20 @@ async function sanitizeNativeStaticLayoutExportOptions( inputPath: await resolveAllowedReadableFilePath(options.inputPath, "Native input"), }; const mutableOptions = sanitized as unknown as Record; + if (sanitized.overlayLayers) { + for (const layer of sanitized.overlayLayers) { + if (typeof layer.path !== "string" || layer.path.trim().length === 0) { + throw new Error(`Native overlay layer ${layer.id} requires a file path`); + } + layer.path = await resolveAllowedReadableFilePath( + layer.path, + `Native overlay ${layer.id}`, + { + mediaOnly: false, + }, + ); + } + } for (const [field, label] of [ ["backgroundImagePath", "Native background image"], @@ -264,6 +277,8 @@ export function registerExportHandlers() { bitrate: number; encodingMode: NativeExportEncodingMode; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: ExportVideoCodec; + encoderPreference?: ExportEncoderPreference; }, ) => { try { @@ -273,13 +288,23 @@ export function registerExportHandlers() { const ffmpegPath = getFfmpegBinaryPath(); const inputMode = options.inputMode ?? "rawvideo"; + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; const sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const outputPath = path.join(app.getPath("temp"), `${sessionId}.mp4`); let encoderName: string; let ffmpegArgs: string[]; - if (inputMode === "h264-stream") { + // Preserve the existing zero-copy H.264 stream-copy path for the default + // H.264 + Auto selection. Explicit hardware/CPU H.264, and any HEVC + // selection, go through the native raw-frame route. + const useH264StreamCopy = + inputMode === "h264-stream" && + videoCodec === "h264" && + encoderPreference === "auto"; + + if (useH264StreamCopy) { // Pre-encoded H.264 Annex B from browser VideoEncoder — just stream-copy into MP4 encoderName = "h264-stream-copy"; ffmpegArgs = buildNativeH264StreamExportArgs({ @@ -287,7 +312,12 @@ export function registerExportHandlers() { outputPath, }); } else { - encoderName = await resolveNativeVideoEncoder(ffmpegPath, options.encodingMode); + encoderName = await resolveNativeVideoEncoder( + ffmpegPath, + options.encodingMode, + videoCodec, + encoderPreference, + ); ffmpegArgs = buildNativeVideoExportArgs(encoderName, options, outputPath); } @@ -317,6 +347,11 @@ export function registerExportHandlers() { writeSequence: Promise.resolve(), sender: event.sender, pendingWriteRequestIds: new Set(), + framePort: null, + framePortReady: false, + nextFrameSequence: 0, + pendingFrameRequests: new Map(), + completedFrameRequestIds: new Set(), completionPromise: new Promise((resolve, reject) => { ffmpegProcess.once("error", (error) => { const processError = @@ -340,6 +375,11 @@ export function registerExportHandlers() { } session.stdinError = stdinError; + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + getNativeVideoExportSessionError(session, stdinError.message), + ); }); ffmpegProcess.once("close", (code, signal) => { if (session.terminating) { @@ -363,7 +403,17 @@ export function registerExportHandlers() { }); }), }; - void session.completionPromise.catch(() => undefined); + void session.completionPromise.catch((error: unknown) => { + const processError = error instanceof Error ? error : new Error(String(error)); + if (!session.processError) { + session.processError = processError; + } + flushNativeVideoExportFramePortPendingRequests( + sessionId, + session, + getNativeVideoExportSessionError(session, processError.message), + ); + }); ffmpegProcess.stderr.on("data", (chunk: Buffer) => { session.stderrOutput += chunk.toString(); @@ -460,14 +510,15 @@ export function registerExportHandlers() { return { success: true, tempPath: result.outputPath, + videoCodec: result.videoCodec, + encoderPreference: result.encoderPreference, + route: result.route, encoderName: primaryBackend === "nvidia-cuda-compositor" ? "nvidia-cuda-compositor" : primaryBackend === "windows-d3d11-compositor" ? "windows-d3d11-compositor" - : result.metrics.chunkCount > 1 - ? "chunked-h264-nvenc" - : "static-layout-h264-nvenc", + : result.encoderName, metrics: result.metrics, }; } catch (error) { @@ -496,6 +547,25 @@ export function registerExportHandlers() { return { success: true }; }); + ipcMain.on("native-video-export-frame-channel", (event, payload: { sessionId?: string }) => { + const port = event.ports[0]; + const sessionId = typeof payload?.sessionId === "string" ? payload.sessionId : ""; + if (!port) { + return; + } + + const session = nativeVideoExportSessions.get(sessionId); + if (!session) { + sendNativeVideoExportFramePortError(port, sessionId, "Invalid native export session", { + fallbackAvailable: true, + }); + port.close(); + return; + } + + attachNativeVideoExportFramePort(sessionId, session, port, event.sender); + }); + ipcMain.on( "native-video-export-write-frames-async", ( @@ -659,6 +729,11 @@ export function registerExportHandlers() { session.outputPath, options ?? {}, ); + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export session finished", + ); nativeVideoExportSessions.delete(sessionId); // Register the finalized path so only app-produced paths can flow back // through finalize-exported-video / discard-exported-temp. @@ -680,6 +755,7 @@ export function registerExportHandlers() { metrics: finalized.metrics, }; } catch (error) { + closeNativeVideoExportFramePort(sessionId, session, String(error)); flushNativeVideoExportPendingWriteRequests(sessionId, session, String(error)); nativeVideoExportSessions.delete(sessionId); await removeTemporaryExportFile(session.outputPath); @@ -805,6 +881,11 @@ export function registerExportHandlers() { session.terminating = true; nativeVideoExportSessions.delete(sessionId); + closeNativeVideoExportFramePort( + sessionId, + session, + "Native video export session was cancelled", + ); flushNativeVideoExportPendingWriteRequests( sessionId, session, diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..d5d851b26 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -21,6 +21,19 @@ export let currentRecordingSession: RecordingSessionData | null = null; // ── Security: approved read paths ───────────────────────────────────────────── export const approvedLocalReadPaths = new Set(); +// Windows paths are case-insensitive and may surface as extended-length +// (`\\?\`) paths from realpath. Fold both forms so policy comparisons do not +// reject a path just because its drive letter or directory casing differs from +// the approved root. Non-Windows paths are compared verbatim. +export function foldPathComparisonKey(filePath: string) { + if (process.platform !== "win32") { + return filePath; + } + + const withoutExtendedPrefix = filePath.replace(/^\\\\\?\\/, "").replace(/^\\.\\/, ""); + return withoutExtendedPrefix.toLowerCase(); +} + // ── Native macOS capture ────────────────────────────────────────────────────── export let nativeScreenRecordingActive = false; export let nativeCaptureProcess: ChildProcessWithoutNullStreams | null = null; @@ -96,6 +109,8 @@ export let cachedNativeMacWindowSourcesAtMs = 0; export let cachedNativeVideoEncoder: { ffmpegPath: string; encodingMode: string; + codec: string; + preference: string; encoderName: string; } | null = null; @@ -283,7 +298,13 @@ export function setCachedNativeMacWindowSourcesAtMs(v: number) { } export function setCachedNativeVideoEncoder( - v: { ffmpegPath: string; encodingMode: string; encoderName: string } | null, + v: { + ffmpegPath: string; + encodingMode: string; + codec: string; + preference: string; + encoderName: string; + } | null, ) { cachedNativeVideoEncoder = v; } diff --git a/electron/native/nvidia-cuda-compositor/CMakeLists.txt b/electron/native/nvidia-cuda-compositor/CMakeLists.txt index e79d9630c..fa2cda64f 100644 --- a/electron/native/nvidia-cuda-compositor/CMakeLists.txt +++ b/electron/native/nvidia-cuda-compositor/CMakeLists.txt @@ -1,5 +1,14 @@ cmake_minimum_required(VERSION 3.24) +# Target the common NVENC-capable architectures with native SASS plus PTX +# fallback. The legacy default (compute_75 only) cannot run on Blackwell +# (sm_120) GPUs. Override with -DCMAKE_CUDA_ARCHITECTURES for exotic targets. +# Must be set before project() so CUDA language init does not pin it to the +# toolkit default. +if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES) + set(CMAKE_CUDA_ARCHITECTURES "75;86;89;90;120") +endif() + project(recordly_nvidia_cuda_compositor LANGUAGES CXX CUDA) set(CMAKE_CXX_STANDARD 17) @@ -7,6 +16,11 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) set(CMAKE_CUDA_STANDARD 17) set(CMAKE_CUDA_STANDARD_REQUIRED ON) +set(RECORDLY_NVENC_HEADERS_DIR + "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/nv-codec-headers/include/ffnvcodec" + CACHE PATH + "Path to the nv-codec-headers include dir (nvEncodeAPI.h 12+/13+ for Blackwell)" +) set(RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../../.tmp/video-sdk-samples" CACHE PATH @@ -27,8 +41,12 @@ add_executable(recordly-nvidia-cuda-compositor ) target_include_directories(recordly-nvidia-cuda-compositor PRIVATE + "${RECORDLY_NVENC_HEADERS_DIR}" "${NVIDIA_SAMPLES_DIR}" "${NVCODEC_DIR}" + "${NVCODEC_DIR}/NvEncoder" + "${NVCODEC_DIR}/NvDecoder" + "${NVCODEC_DIR}/../Utils" ) target_compile_definitions(recordly-nvidia-cuda-compositor PRIVATE diff --git a/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs b/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs new file mode 100644 index 000000000..fa83b86e5 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/cursorTelemetry.mjs @@ -0,0 +1,191 @@ +// Cursor telemetry contract shared by the NVIDIA CUDA compositor wrapper and +// its callers. +// +// The canonical `--cursor-json` payload is a JSON object with a `samples` +// array; each sample is +// { timeMs, cx, cy, cursorType?, cursorTypeIndex?, interactionType?, +// bounceScale?, visible? } +// The exact generated telemetry row format (TSV inside the pipeline, CSV from +// the Windows GPU compositor telemetry prep) is +// timeMscxcycursorTypeIndexbounceScalevisible +// (6 whitespace- or comma-separated fields). Parsing must NEVER hand raw rows +// to JSON.parse; a mis-wired caller that points --cursor-json at a CSV/TSV +// samples file must be accepted or rejected with an actionable error, not a +// SyntaxError. + +import { writeFileSync } from "node:fs"; + +export const CURSOR_SAMPLE_TYPES = [ + "arrow", + "text", + "pointer", + "crosshair", + "open-hand", + "closed-hand", + "resize-ew", + "resize-ns", + "not-allowed", +]; +export const cursorTypeIndexes = new Map(CURSOR_SAMPLE_TYPES.map((type, index) => [type, index])); +export const CURSOR_SAMPLE_MAX_TYPE_INDEX = CURSOR_SAMPLE_TYPES.length - 1; + +const CLICK_TYPES = new Set(["click", "double-click", "right-click", "middle-click"]); +const DEFAULT_BOUNCE_DURATION_MS = 180; + +function isCursorClickType(interactionType) { + return CLICK_TYPES.has(interactionType); +} + +export function cursorBounceScale(interactionType, ageMs, durationMs = DEFAULT_BOUNCE_DURATION_MS) { + if (!isCursorClickType(interactionType)) { + return 1; + } + if (ageMs < 0 || ageMs > durationMs) { + return 1; + } + const progress = 1 - ageMs / durationMs; + return Math.max(0.72, 1 - Math.sin(progress * Math.PI) * 0.08); +} + +export function latestClickSample(samples, sampleIndex) { + for (let index = sampleIndex; index >= 0; index -= 1) { + const sample = samples[index]; + if (sample && isCursorClickType(sample.interactionType)) { + return sample; + } + } + return null; +} + +export function isValidCursorSample(sample) { + return ( + sample !== null && + typeof sample === "object" && + Number.isFinite(sample.timeMs) && + Number.isFinite(sample.cx) && + Number.isFinite(sample.cy) + ); +} + +function normalizeCursorTypeIndex(value) { + if (typeof value === "string" && cursorTypeIndexes.has(value)) { + return cursorTypeIndexes.get(value); + } + if (Number.isFinite(value)) { + return Math.max(0, Math.min(CURSOR_SAMPLE_MAX_TYPE_INDEX, Math.round(value))); + } + return 0; +} + +// Formats samples into the exact TSV sidecar consumed by the native compositor +// (`--cursor-samples`): timeMs, cx, cy, cursorTypeIndex, bounceScale, visible, +// tab-separated, one row per line, in input order. Preserves renderer-resolved +// click bounce (bounceScale) and click type when present. +export function formatCursorSamplesTsv(samples) { + const rows = []; + for (let index = 0; index < samples.length; index += 1) { + const sample = samples[index]; + if (!isValidCursorSample(sample)) { + continue; + } + const clickSample = latestClickSample(samples, index); + const bounceScale = Number.isFinite(sample.bounceScale) + ? sample.bounceScale + : clickSample + ? cursorBounceScale(clickSample.interactionType, sample.timeMs - clickSample.timeMs) + : 1; + rows.push( + [ + sample.timeMs, + sample.cx, + sample.cy, + normalizeCursorTypeIndex(sample.cursorTypeIndex), + Number(bounceScale.toFixed(4)), + sample.visible === false ? 0 : 1, + ].join("\t"), + ); + } + return rows.join("\n"); +} + +export function writeCursorSamplesFile(samples, outputPath) { + const lines = formatCursorSamplesTsv(samples); + writeFileSync(outputPath, lines ? `${lines}\n` : ""); + return samples.length; +} + +function clampUnit(value) { + return Math.min(1, Math.max(0, value)); +} + +function parseCursorTelemetryRows(text, sourcePath) { + const samples = []; + const lines = text.split(/\r?\n/); + for (let lineIndex = 0; lineIndex < lines.length; lineIndex += 1) { + const line = lines[lineIndex].trim(); + if (!line) { + continue; + } + let fields = line.split("\t"); + if (fields.length === 1) { + fields = line.split(","); + } + if (fields.length < 3 || fields.length > 6) { + throw new Error( + `${sourcePath} line ${lineIndex + 1} is not a cursor telemetry row; expected 6 TSV/CSV fields (timeMs,cx,cy,cursorTypeIndex,bounceScale,visible), received ${fields.length}: ${line}`, + ); + } + const numbers = fields.map(Number); + if (!numbers.slice(0, 3).every(Number.isFinite)) { + throw new Error( + `${sourcePath} line ${lineIndex + 1} has invalid cursor position fields: ${line}`, + ); + } + samples.push({ + timeMs: Math.max(0, numbers[0]), + cx: clampUnit(numbers[1]), + cy: clampUnit(numbers[2]), + cursorTypeIndex: normalizeCursorTypeIndex(numbers[3]), + bounceScale: Number.isFinite(numbers[4]) ? Math.max(0.1, Math.min(2, numbers[4])) : 1, + visible: Number.isFinite(numbers[5]) ? numbers[5] !== 0 : true, + }); + } + if (samples.length === 0) { + throw new Error( + `${sourcePath} contains no parseable cursor telemetry samples; expected a JSON {"samples":[...]} payload or TSV/CSV rows (timeMs,cx,cy,cursorTypeIndex,bounceScale,visible)`, + ); + } + return samples; +} + +// Parses a --cursor-json file into cursor samples. Accepts the canonical JSON +// payload ({"samples":[...]} or a bare array) and the exact generated TSV/CSV +// row format so a mis-wired CSV/TSV telemetry file never reaches JSON.parse. +export function parseCursorTelemetrySamples(text, sourcePath = "cursor telemetry") { + if (typeof text !== "string") { + throw new Error(`${sourcePath} must contain text content`); + } + const trimmed = text.trim(); + if (!trimmed) { + return []; + } + + let payload = null; + let jsonError = null; + try { + payload = JSON.parse(trimmed); + } catch (error) { + jsonError = error; + } + if (jsonError === null) { + const samples = Array.isArray(payload) ? payload : payload?.samples; + if (Array.isArray(samples)) { + return samples.filter((sample) => isValidCursorSample(sample)); + } + throw new Error( + `${sourcePath} is valid JSON but does not contain a samples array; expected {"samples":[...]}`, + ); + } + + return parseCursorTelemetryRows(trimmed, sourcePath); +} diff --git a/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs b/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs new file mode 100644 index 000000000..4ee8e747b --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/cursorTelemetry.test.mjs @@ -0,0 +1,169 @@ +import { describe, expect, it } from "vitest"; +import { + formatCursorSamplesTsv, + parseCursorTelemetrySamples, + writeCursorSamplesFile, +} from "./cursorTelemetry.mjs"; + +// Regression guard for the CUDA compositor contract bug where a raw CSV/TSV +// cursor telemetry file was handed to JSON.parse (run-mp4-pipeline.mjs +// "JSON.parse receives 0,0.523828,0.731944,0,1,1"). The parser must accept the +// exact generated telemetry row format without ever JSON.parsing raw rows. + +const CSV_TELEMETRY = [ + "0,0.523828,0.731944,0,1,1", + "1000,0.62,0.7,2,0.9784,1", + "2000,0.3,0.4,1,0.8675,0", +].join("\n"); + +const JSON_TELEMETRY = JSON.stringify({ + samples: [ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { timeMs: 1000, cx: 0.62, cy: 0.7, cursorTypeIndex: 2, bounceScale: 0.9784, visible: true }, + { timeMs: 2000, cx: 0.3, cy: 0.4, cursorTypeIndex: 1, bounceScale: 0.8675, visible: false }, + ], +}); + +describe("parseCursorTelemetrySamples", () => { + it("parses the exact generated CSV row format without JSON.parse", () => { + expect(() => JSON.parse(CSV_TELEMETRY)).toThrow(); + + const samples = parseCursorTelemetrySamples(CSV_TELEMETRY, "cursor-telemetry.csv"); + expect(samples).toEqual([ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { + timeMs: 1000, + cx: 0.62, + cy: 0.7, + cursorTypeIndex: 2, + bounceScale: 0.9784, + visible: true, + }, + { + timeMs: 2000, + cx: 0.3, + cy: 0.4, + cursorTypeIndex: 1, + bounceScale: 0.8675, + visible: false, + }, + ]); + }); + + it("parses the exact generated TSV row format (pipeline cursor sidecar)", () => { + const tsv = ["0\t0.523828\t0.731944\t0\t1\t1", "1000\t0.62\t0.7\t2\t0.9784\t1"].join("\n"); + const samples = parseCursorTelemetrySamples(tsv, "cursor.tsv"); + expect(samples).toEqual([ + { + timeMs: 0, + cx: 0.523828, + cy: 0.731944, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }, + { + timeMs: 1000, + cx: 0.62, + cy: 0.7, + cursorTypeIndex: 2, + bounceScale: 0.9784, + visible: true, + }, + ]); + }); + + it("parses the canonical JSON payload with a samples array", () => { + const samples = parseCursorTelemetrySamples(JSON_TELEMETRY, "cursor-telemetry.json"); + expect(samples).toHaveLength(3); + expect(samples[1]).toMatchObject({ timeMs: 1000, cx: 0.62, cursorTypeIndex: 2 }); + expect(samples[2].visible).toBe(false); + }); + + it("accepts a bare JSON array of samples", () => { + const samples = parseCursorTelemetrySamples( + JSON.stringify(JSON.parse(JSON_TELEMETRY).samples), + ); + expect(samples).toHaveLength(3); + }); + + it("clamps row values to the telemetry contract bounds", () => { + const samples = parseCursorTelemetrySamples("0,-0.5,1.7,99,5,0", "cursor.csv"); + expect(samples[0]).toEqual({ + timeMs: 0, + cx: 0, + cy: 1, + cursorTypeIndex: 8, + bounceScale: 2, + visible: false, + }); + }); + + it("accepts partial 3-field rows with defaults like the native loader", () => { + const samples = parseCursorTelemetrySamples("0\t0.5\t0.5", "cursor.tsv"); + expect(samples[0]).toEqual({ + timeMs: 0, + cx: 0.5, + cy: 0.5, + cursorTypeIndex: 0, + bounceScale: 1, + visible: true, + }); + }); + + it("round-trips the generated row format without value drift", () => { + const parsed = parseCursorTelemetrySamples(CSV_TELEMETRY, "cursor-telemetry.csv"); + const tsv = formatCursorSamplesTsv(parsed); + const reparsed = parseCursorTelemetrySamples(tsv, "roundtrip.tsv"); + expect(reparsed).toEqual(parsed); + }); + + it("rejects non-telemetry text with an actionable error instead of JSON.parse noise", () => { + expect(() => parseCursorTelemetrySamples("hello world", "bad.txt")).toThrow( + /bad\.txt line 1 is not a cursor telemetry row/, + ); + expect(parseCursorTelemetrySamples("", "empty.csv")).toEqual([]); + }); + + it("rejects valid JSON that is not a samples payload", () => { + expect(() => parseCursorTelemetrySamples('{"layers":[]}', "overlay.json")).toThrow( + /does not contain a samples array/, + ); + }); +}); + +describe("writeCursorSamplesFile", () => { + it("writes the TSV sidecar consumed by the native compositor", async () => { + const os = await import("node:os"); + const path = await import("node:path"); + const fs = await import("node:fs"); + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "recordly-cursor-telemetry-")); + const outputPath = path.join(dir, "cursor.tsv"); + try { + const samples = parseCursorTelemetrySamples(JSON_TELEMETRY, "cursor-telemetry.json"); + const count = writeCursorSamplesFile(samples, outputPath); + expect(count).toBe(3); + const written = fs.readFileSync(outputPath, "utf8"); + expect(written).toBe( + "0\t0.523828\t0.731944\t0\t1\t1\n1000\t0.62\t0.7\t2\t0.9784\t1\n2000\t0.3\t0.4\t1\t0.8675\t0\n", + ); + expect(parseCursorTelemetrySamples(written, outputPath)).toEqual(samples); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs new file mode 100644 index 000000000..b14081c3a --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs @@ -0,0 +1,106 @@ +// Renderer-prepared transparent RGBA overlay sidecar manifest reader for the +// NVIDIA CUDA compositor wrapper (run-mp4-pipeline.mjs). +// +// Manifest layers carry a logical frameCount plus an optional +// effectiveFrameCount. When renderer-side deduplication truncated an identical +// suffix, the sidecar physically stores only effectiveFrameCount frames +// (1 <= effectiveFrameCount <= frameCount) and the final physical frame repeats +// for output indices [effectiveFrameCount, frameCount). Byte validation and the +// native --overlay descriptor must therefore use the physical frame count while +// the returned metadata/summary keeps the logical count. Manifests without +// effectiveFrameCount are fully dynamic layers and behave exactly as before. + +import { existsSync, readFileSync, statSync } from "node:fs"; +import { resolve } from "node:path"; + +function fail(message) { + throw new Error(message); +} + +export function readOverlayManifest(manifestPath, outputSize) { + if (!manifestPath) { + return []; + } + const { outputWidth, outputHeight } = outputSize; + const resolvedPath = resolve(manifestPath); + if (!existsSync(resolvedPath)) { + fail(`Overlay manifest does not exist: ${resolvedPath}`); + } + + let manifest; + try { + manifest = JSON.parse(readFileSync(resolvedPath, "utf8")); + } catch (error) { + fail(`Invalid overlay manifest ${resolvedPath}: ${error.message}`); + } + if (!Array.isArray(manifest.layers)) { + fail(`Overlay manifest requires a layers array: ${resolvedPath}`); + } + + const layers = []; + for (const layer of manifest.layers) { + const id = typeof layer?.id === "string" ? layer.id : ""; + const layerPath = typeof layer?.path === "string" ? layer.path : ""; + const x = Number(layer?.x); + const y = Number(layer?.y); + const width = Number(layer?.width); + const height = Number(layer?.height); + const frameCount = Number(layer?.frameCount); + const effectiveFrameCount = + layer?.effectiveFrameCount === undefined || layer?.effectiveFrameCount === null + ? null + : Number(layer.effectiveFrameCount); + if (!id || !layerPath) { + fail(`Overlay manifest layer requires an id and path: ${resolvedPath}`); + } + if ( + ![x, y, width, height, frameCount].every(Number.isSafeInteger) || + width <= 0 || + height <= 0 || + frameCount <= 0 || + x < 0 || + y < 0 + ) { + fail(`Invalid overlay manifest layer ${id}: ${resolvedPath}`); + } + if (effectiveFrameCount !== null) { + // Mirror the renderer contract (validateNativeStaticLayoutOverlayLayer): + // the physical sidecar count must be a positive integer no greater than + // the logical count. Malformed values fail here with the same generic + // invalid-layer message instead of a confusing truncation error. + if ( + !Number.isSafeInteger(effectiveFrameCount) || + effectiveFrameCount < 1 || + effectiveFrameCount > frameCount + ) { + fail(`Invalid overlay manifest layer ${id}: ${resolvedPath}`); + } + } + if (x + width > outputWidth || y + height > outputHeight) { + fail(`Overlay layer ${id} exceeds the output canvas: ${resolvedPath}`); + } + const resolvedLayerPath = resolve(layerPath); + if (!existsSync(resolvedLayerPath)) { + fail(`Overlay layer ${id} does not exist: ${resolvedLayerPath}`); + } + const physicalFrameCount = effectiveFrameCount ?? frameCount; + const expectedBytes = width * height * 4 * physicalFrameCount; + const stat = statSync(resolvedLayerPath); + if (stat.size < expectedBytes) { + fail( + `Overlay layer ${id} is truncated: expected at least ${expectedBytes} bytes, received ${stat.size}`, + ); + } + layers.push({ + id, + path: resolvedLayerPath, + x, + y, + width, + height, + frameCount, + ...(effectiveFrameCount !== null ? { effectiveFrameCount } : {}), + }); + } + return layers; +} diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs new file mode 100644 index 000000000..df0b2db85 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs @@ -0,0 +1,294 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, expect, it } from "vitest"; +import { readOverlayManifest } from "./overlayManifest.mjs"; + +// Contract for renderer-prepared transparent RGBA overlay sidecars: +// - frameCount is the logical frame count (durationSec * frameRate). +// - effectiveFrameCount (optional) is the physical frame count when renderer +// deduplication truncated an identical suffix; the sidecar then stores only +// effectiveFrameCount frames and the final physical frame repeats for output +// indices [effectiveFrameCount, frameCount). +// - Byte validation must use the physical count; metadata must keep the logical +// count; the native --overlay descriptor must receive the physical count so +// OverlayFrameSource clamps/repeats the final frame. + +const OUTPUT_SIZE = { outputWidth: 1920, outputHeight: 1080 }; +const FRAME_BYTES = 4 * 4 * 4; // 4x4 RGBA + +function makeTempDir(prefix = "recordly-overlay-manifest-") { + return mkdtempSync(join(tmpdir(), prefix)); +} + +function writeManifest(dir, layers, name = "overlay-manifest.json") { + const manifestPath = join(dir, name); + writeFileSync(manifestPath, JSON.stringify({ layers })); + return manifestPath; +} + +function writeSidecar(dir, byteLength, name = "overlay.rgba") { + const sidecarPath = join(dir, name); + writeFileSync(sidecarPath, Buffer.alloc(byteLength, 0x7f)); + return sidecarPath; +} + +function layer(overrides = {}) { + return { + id: "overlay-a", + path: "", + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + ...overrides, + }; +} + +describe("readOverlayManifest", () => { + it("returns an empty array when no manifest path is provided", () => { + expect(readOverlayManifest("", OUTPUT_SIZE)).toEqual([]); + expect(readOverlayManifest(null, OUTPUT_SIZE)).toEqual([]); + }); + + it("rejects a missing manifest file", () => { + expect(() => readOverlayManifest("/missing/overlay.json", OUTPUT_SIZE)).toThrow( + "Overlay manifest does not exist:", + ); + }); + + it("rejects invalid JSON and a missing layers array", () => { + const dir = makeTempDir(); + try { + const badJson = join(dir, "bad.json"); + writeFileSync(badJson, "{not json"); + expect(() => readOverlayManifest(badJson, OUTPUT_SIZE)).toThrow( + /Invalid overlay manifest/, + ); + + const noLayers = join(dir, "no-layers.json"); + writeFileSync(noLayers, JSON.stringify({ frames: 10 })); + expect(() => readOverlayManifest(noLayers, OUTPUT_SIZE)).toThrow( + "Overlay manifest requires a layers array:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts a manifest without effectiveFrameCount and validates bytes against the logical count", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar })]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toEqual([ + { + id: "overlay-a", + path: sidecar, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts an effectiveFrameCount sidecar with only physical frames and preserves the logical count", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 2); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toEqual([ + { + id: "overlay-a", + path: sidecar, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + effectiveFrameCount: 2, + }, + ]); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts effectiveFrameCount equal to frameCount (no dedup truncation)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 10 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].frameCount).toBe(10); + expect(layers[0].effectiveFrameCount).toBe(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("treats a null effectiveFrameCount as absent (backward compatible)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, effectiveFrameCount: null }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0]).not.toHaveProperty("effectiveFrameCount"); + expect(layers[0].frameCount).toBe(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects effectiveFrameCount values outside 1..frameCount with the invalid-layer message", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + for (const effectiveFrameCount of [0, -1, 11, 1.5, "not-a-number"]) { + const manifestPath = writeManifest( + dir, + [layer({ path: sidecar, frameCount: 10, effectiveFrameCount })], + `invalid-${String(effectiveFrameCount)}.json`, + ); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + } + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("validates physical bytes against effectiveFrameCount, not the logical count", () => { + const dir = makeTempDir(); + try { + // Logical count implies 10 frames (10 * FRAME_BYTES) but the sidecar + // physically stores 2; with the physical-count rule this is valid. + const sidecar = writeSidecar(dir, FRAME_BYTES * 2); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + expect(readOverlayManifest(manifestPath, OUTPUT_SIZE)).toHaveLength(1); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("fails a sidecar truncated below the physical count (with effectiveFrameCount)", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 2 - 1); + const manifestPath = writeManifest(dir, [ + layer({ path: sidecar, frameCount: 10, effectiveFrameCount: 2 }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Overlay layer overlay-a is truncated: expected at least ${ + FRAME_BYTES * 2 + } bytes, received ${FRAME_BYTES * 2 - 1}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps the backward-compatible truncation message for manifests without effectiveFrameCount", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 9); + const manifestPath = writeManifest(dir, [layer({ path: sidecar })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Overlay layer overlay-a is truncated: expected at least ${ + FRAME_BYTES * 10 + } bytes, received ${FRAME_BYTES * 9}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("enforces output-canvas bounds for every layer", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar, x: 1918, width: 4 })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Overlay layer overlay-a exceeds the output canvas:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects layers missing an id/path or with invalid geometry", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const missingId = writeManifest(dir, [layer({ path: sidecar, id: "" })]); + expect(() => readOverlayManifest(missingId, OUTPUT_SIZE)).toThrow( + "Overlay manifest layer requires an id and path:", + ); + + const missingPath = writeManifest(dir, [layer({ path: "" })]); + expect(() => readOverlayManifest(missingPath, OUTPUT_SIZE)).toThrow( + "Overlay manifest layer requires an id and path:", + ); + + const badGeometry = writeManifest(dir, [layer({ path: sidecar, width: 0 })]); + expect(() => readOverlayManifest(badGeometry, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + + const negativeX = writeManifest(dir, [layer({ path: sidecar, x: -1 })]); + expect(() => readOverlayManifest(negativeX, OUTPUT_SIZE)).toThrow( + "Invalid overlay manifest layer overlay-a:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a missing sidecar file", () => { + const dir = makeTempDir(); + try { + const manifestPath = writeManifest(dir, [layer({ path: join(dir, "nope.rgba") })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Overlay layer overlay-a does not exist:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("handles mixed layers (deduped and fully dynamic) in one manifest", () => { + const dir = makeTempDir(); + try { + const fullSidecar = writeSidecar(dir, FRAME_BYTES * 10, "full.rgba"); + const dedupedSidecar = writeSidecar(dir, FRAME_BYTES * 3, "deduped.rgba"); + const manifestPath = writeManifest(dir, [ + layer({ id: "a", path: fullSidecar, frameCount: 10 }), + layer({ id: "b", path: dedupedSidecar, frameCount: 10, effectiveFrameCount: 3 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "a", frameCount: 10 }); + expect(layers[0]).not.toHaveProperty("effectiveFrameCount"); + expect(layers[1]).toMatchObject({ id: "b", frameCount: 10, effectiveFrameCount: 3 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs index 43324b051..0b3832d0e 100644 --- a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs +++ b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs @@ -48,18 +48,9 @@ function resolveToolCommand(envNames, moduleName, fallbackName) { const ffmpegCommand = resolveToolCommand(["RECORDLY_FFMPEG_EXE"], "ffmpeg-static", "ffmpeg"); const ffprobeCommand = resolveToolCommand(["RECORDLY_FFPROBE_EXE"], "ffprobe-static", "ffprobe"); -const cursorTypes = [ - "arrow", - "text", - "pointer", - "crosshair", - "open-hand", - "closed-hand", - "resize-ew", - "resize-ns", - "not-allowed", -]; -const cursorTypeIndexes = new Map(cursorTypes.map((type, index) => [type, index])); +import { parseCursorTelemetrySamples, writeCursorSamplesFile } from "./cursorTelemetry.mjs"; +import { readOverlayManifest } from "./overlayManifest.mjs"; +import { shouldProbeSourcePts } from "./sourcePtsPlan.mjs"; function fail(message) { throw new Error(message); @@ -284,6 +275,7 @@ function emitPreparationProgress(totalFrames, percentage, stage) { const progressStage = stage ?? "preparing"; const finalizing = progressStage === "finalizing"; const payload = { + outputCodec, currentFrame: finalizing ? Math.max(1, Math.floor(totalFrames)) : 0, totalFrames: Math.max(1, Math.floor(totalFrames)), percentage: Number(Math.min(99, Math.max(0, percentage)).toFixed(2)), @@ -451,69 +443,6 @@ function summarizeGpuSamples(samples) { return summary; } -function cursorBounceScale(interactionType, ageMs, durationMs = 180) { - if (!["click", "double-click", "right-click", "middle-click"].includes(interactionType)) { - return 1; - } - if (ageMs < 0 || ageMs > durationMs) { - return 1; - } - const progress = 1 - ageMs / durationMs; - return Math.max(0.72, 1 - Math.sin(progress * Math.PI) * 0.08); -} - -function latestClickSample(samples, sampleIndex) { - for (let index = sampleIndex; index >= 0; index -= 1) { - const sample = samples[index]; - if ( - ["click", "double-click", "right-click", "middle-click"].includes( - sample?.interactionType, - ) - ) { - return sample; - } - } - return null; -} - -function writeCursorSamples(cursorPayload, outputPath) { - const samples = Array.isArray(cursorPayload.samples) ? cursorPayload.samples : []; - const cursorLines = samples - .map((sample, index) => { - if ( - !Number.isFinite(sample?.timeMs) || - !Number.isFinite(sample?.cx) || - !Number.isFinite(sample?.cy) - ) { - return null; - } - const clickSample = latestClickSample(samples, index); - const bounceScale = Number.isFinite(sample.bounceScale) - ? sample.bounceScale - : clickSample - ? cursorBounceScale( - clickSample.interactionType, - sample.timeMs - clickSample.timeMs, - ) - : 1; - return [ - sample.timeMs, - sample.cx, - sample.cy, - cursorTypeIndexes.get(sample.cursorType) ?? - (Number.isFinite(sample.cursorTypeIndex) - ? Math.max(0, Math.min(8, Math.round(sample.cursorTypeIndex))) - : 0), - Number(bounceScale.toFixed(4)), - sample.visible === false ? 0 : 1, - ].join("\t"); - }) - .filter(Boolean) - .join("\n"); - writeFileSync(outputPath, cursorLines ? `${cursorLines}\n` : ""); - return samples.length; -} - function renderTahoeCursorAtlas(workDir) { const rgbaPath = join(workDir, "tahoe-cursor-atlas.rgba"); const metadataPath = join(workDir, "tahoe-cursor-atlas.tsv"); @@ -698,7 +627,11 @@ function getVideoInfo(inputPath) { fail(`No video stream found in ${inputPath}`); } if (stream.codec_name !== "h264") { - fail(`The NVIDIA CUDA compositor currently expects H.264 input, got ${stream.codec_name}`); + fail( + `The NVIDIA CUDA compositor only supports H.264 input video; got ${ + stream.codec_name ?? "unknown" + }`, + ); } const durationSec = Number(stream.duration); if (!Number.isFinite(durationSec) || durationSec <= 0) { @@ -930,6 +863,11 @@ const inputPath = resolve(getArg("--input")); const outputPath = resolve( getArg("--output", join(scriptDir, "recordly-nvdec-nvenc-mp4-output.mp4")), ); +const outputCodec = getArg("--output-codec", "h264"); +if (!["h264", "hevc"].includes(outputCodec)) { + throw new Error(`Unsupported --output-codec: ${outputCodec}; expected h264 or hevc`); +} +const elementaryStreamFormat = outputCodec; const requestedOutputWidth = Math.round(getNumberArg("--width", 0)); const requestedOutputHeight = Math.round(getNumberArg("--height", 0)); const fps = Math.round(getNumberArg("--fps", 30)); @@ -982,6 +920,10 @@ const cursorAtlasPng = getArg("--cursor-atlas-png", ""); const cursorAtlasMetadata = getArg("--cursor-atlas-metadata", ""); const zoomTelemetry = getArg("--zoom-telemetry", ""); const timelineMap = getArg("--timeline-map", ""); +const overlayManifest = getArg("--overlay-manifest", ""); +const temporalBlurSampleCount = getNumberArg("--temporal-blur-sample-count", 0); +const temporalBlurShutterFraction = getNumberArg("--temporal-blur-shutter-fraction", 0); +const temporalBlurWeightPower = getNumberArg("--temporal-blur-weight-power", 1); if (!existsSync(inputPath)) { fail(`Input does not exist: ${inputPath}`); @@ -1028,7 +970,7 @@ const webcamBaseName = webcamInput const annexBPath = join(workDir, `${baseName}.annexb.h264`); const webcamAnnexBPath = join(workDir, `${webcamBaseName}.annexb.h264`); const cursorSamplesPath = join(workDir, `${baseName}.cursor.tsv`); -const encodedPath = join(workDir, `${baseName}.mapped-callback.h264`); +const encodedPath = join(workDir, `${baseName}.mapped-callback.${outputCodec}`); const shouldBakeStaticShadow = Boolean(backgroundImage) && contentWidth > 0 && @@ -1235,11 +1177,26 @@ const demuxPromise = endPercentage: 2, }, ); -const sourcePtsPromise = writeFramePtsSidecarAsync(inputPath, sourceDurationSec, sourcePtsPath); +// Source frame PTS is only required for timeline-map exports and for inline +// audio mux validation (the wrapper checks the native summary reports a +// timestamp-aligned mode before trusting the muxed audio). For plain video-only +// exports the per-packet ffprobe scan is pure overhead (it dominates the wall +// time of short 4K exports), so skip it unless it is actually consumed. +const needsSourcePts = shouldProbeSourcePts({ + hasTimelineSegments: timelineSegments.length > 0, + videoOnly, + forceSourcePts: process.env.RECORDLY_NVIDIA_CUDA_FORCE_SOURCE_PTS, +}); +const sourcePtsPromise = needsSourcePts + ? writeFramePtsSidecarAsync(inputPath, sourceDurationSec, sourcePtsPath) + : Promise.resolve(zeroElapsed()); if (cursorJson) { - const cursorPayload = JSON.parse(readFileSync(resolve(cursorJson), "utf8")); - writeCursorSamples(cursorPayload, cursorSamplesPath); + const cursorPayload = parseCursorTelemetrySamples( + readFileSync(resolve(cursorJson), "utf8"), + resolve(cursorJson), + ); + writeCursorSamplesFile(cursorPayload, cursorSamplesPath); } const cursorAtlas = cursorJson && cursorHeight > 0 && cursorAtlasPng && cursorAtlasMetadata @@ -1274,6 +1231,8 @@ const encodeArgs = [ annexBPath, "--output", encodedPath, + "--output-codec", + outputCodec, "--fps", String(fps), "--input-frames", @@ -1402,6 +1361,34 @@ if (contentWidth > 0 && contentHeight > 0) { if (zoomTelemetry) { encodeArgs.push("--zoom-samples", resolve(zoomTelemetry)); } +if (temporalBlurSampleCount >= 3) { + encodeArgs.push( + "--temporal-blur-sample-count", + String(temporalBlurSampleCount), + "--temporal-blur-shutter-fraction", + String(temporalBlurShutterFraction), + "--temporal-blur-weight-power", + String(temporalBlurWeightPower), + ); +} +const overlayLayers = readOverlayManifest(overlayManifest, { outputWidth, outputHeight }); +if (overlayLayers.length) { + for (const layer of overlayLayers) { + encodeArgs.push( + "--overlay", + layer.path, + String(layer.x), + String(layer.y), + String(layer.width), + String(layer.height), + // The native OverlayFrameSource clamps/repeats the final physical frame + // for output indices beyond the physical count, so the descriptor must + // carry the physical sidecar count (effectiveFrameCount when renderer + // dedup truncated an identical suffix, otherwise the logical count). + String(layer.effectiveFrameCount ?? layer.frameCount), + ); + } +} const encode = reuseIntermediates && existsSync(encodedPath) ? { elapsedMs: 0, stdout: "", gpuSummary: null } @@ -1411,7 +1398,18 @@ const encode = sampleGpuDuringEncode ? gpuSampleIntervalMs : 0, ); const nativeSummary = encode.stdout ? parseProbeSummary(encode.stdout) : null; +if (nativeSummary?.outputCodec && nativeSummary.outputCodec !== outputCodec) { + fail(`Native output codec mismatch: expected ${outputCodec}, got ${nativeSummary.outputCodec}`); +} +const elementaryStreamInputArgs = [ + "-f", + elementaryStreamFormat, + "-framerate", + String(fps), + "-i", + encodedPath, +]; const mux = skipMux ? { elapsedMs: 0 } : videoOnly @@ -1423,10 +1421,7 @@ const mux = skipMux "-loglevel", "error", "-stats", - "-framerate", - String(fps), - "-i", - encodedPath, + ...elementaryStreamInputArgs, "-map", "0:v:0", "-c:v", @@ -1449,10 +1444,7 @@ const mux = skipMux "-loglevel", "error", "-stats", - "-framerate", - String(fps), - "-i", - encodedPath, + ...elementaryStreamInputArgs, "-i", inputPath, "-map", @@ -1487,6 +1479,13 @@ const outputInfo = skipMux const outputStreams = outputInfo.streams ?? []; const outputVideo = outputStreams.find((stream) => stream.codec_type === "video") ?? null; const outputAudio = outputStreams.find((stream) => stream.codec_type === "audio") ?? null; +if (!skipMux && outputVideo?.codec_name !== outputCodec) { + fail( + `Muxed output codec mismatch: expected ${outputCodec}, got ${ + outputVideo?.codec_name ?? "none" + }`, + ); +} console.log( JSON.stringify( @@ -1497,6 +1496,8 @@ console.log( requestedOutputPath: outputPath, encodedPath, fps, + outputCodec, + elementaryStreamFormat, bitrateMbps, encodingMode, streamSync, @@ -1566,6 +1567,24 @@ console.log( inputPath: resolve(zoomTelemetry), } : null, + overlay: overlayLayers.length + ? { + layers: overlayLayers.map((layer) => ({ + id: layer.id, + path: layer.path, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + ...(layer.effectiveFrameCount !== undefined + ? { effectiveFrameCount: layer.effectiveFrameCount } + : {}), + physicalFrameCount: + layer.effectiveFrameCount ?? layer.frameCount, + })), + } + : null, } : null, gpuSampleIntervalMs: sampleGpuDuringEncode ? gpuSampleIntervalMs : null, diff --git a/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs new file mode 100644 index 000000000..5b10ffe4f --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.mjs @@ -0,0 +1,14 @@ +// Source PTS sidecar decision for the CUDA export pipeline. +// +// The per-packet ffprobe scan that writes the frame PTS sidecar dominates the +// wall time of short 4K exports (~1.7s per 6s clip). It is only consumed when +// a timeline map is present (mapped-callback frame selection needs source PTS) +// or when the wrapper will inline-mux audio (the wrapper validates the native +// summary reports a timestamp-aligned mode before trusting the muxed audio). +// Plain video-only exports with no timeline skip the probe entirely; the native +// compositor's decoder-policy frame selection produces the same output frames. + +export function shouldProbeSourcePts(options) { + const { hasTimelineSegments, videoOnly, forceSourcePts } = options; + return hasTimelineSegments === true || videoOnly !== true || forceSourcePts === "1"; +} diff --git a/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs new file mode 100644 index 000000000..229381ec6 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/sourcePtsPlan.test.mjs @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { shouldProbeSourcePts } from "./sourcePtsPlan.mjs"; + +describe("shouldProbeSourcePts", () => { + it("probes when a timeline map is present (mapped-callback needs source PTS)", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: true, + videoOnly: true, + forceSourcePts: undefined, + }), + ).toBe(true); + }); + + it("probes when the wrapper will inline-mux audio (non-video-only)", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: false, + forceSourcePts: undefined, + }), + ).toBe(true); + }); + + it("skips the probe for plain video-only exports without a timeline", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: true, + forceSourcePts: undefined, + }), + ).toBe(false); + }); + + it("honors the force override for diagnostics", () => { + expect( + shouldProbeSourcePts({ + hasTimelineSegments: false, + videoOnly: true, + forceSourcePts: "1", + }), + ).toBe(true); + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/src/main.cu b/electron/native/nvidia-cuda-compositor/src/main.cu index 28c72123d..e691b4974 100644 --- a/electron/native/nvidia-cuda-compositor/src/main.cu +++ b/electron/native/nvidia-cuda-compositor/src/main.cu @@ -2,19 +2,24 @@ #include #include +#include #include #include +#include #include #include #include +#include #include #include #include #include #include +#include #include #include #include +#include #include #include "NvDecoder/NvDecoder.h" @@ -33,9 +38,29 @@ struct TimelineSegment { double speed = 1.0; }; +// Renderer-prepared transparent RGBA overlay sidecar layer. The sidecar is a +// raw top-down RGBA stream with frameCount frames of width*height*4 bytes at +// the export frame rate. Layers are composited in the order they appear in +// options.overlayLayers (z-order) after the video layout and zoom blur, which +// matches the renderer contract (overlays are drawn above the blurred video). +struct OverlayLayerDescriptor { + std::string path; + int x = 0; + int y = 0; + int width = 0; + int height = 0; + int frameCount = 0; +}; + +enum class OutputCodec { + H264, + HEVC, +}; + struct Options { std::string inputPath; std::string outputPath = "recordly-nvidia-cuda-compositor.h264"; + OutputCodec outputCodec = OutputCodec::H264; std::string sourcePtsPath; std::string timelineMapPath; std::vector timelineSegments; @@ -87,15 +112,46 @@ struct Options { int cursorAtlasWidth = 0; int cursorAtlasHeight = 0; std::string zoomSamplesPath; + // Renderer-resolved temporal zoom motion blur plan (see temporalMotionBlur.ts): + // the compositor derives per-frame sample offsets/weights from these three + // values plus the output frame duration. 0 sample count disables temporal + // blur so the existing spatial blur telemetry path is used. + int temporalBlurSampleCount = 0; + double temporalBlurShutterFraction = 0.0; + double temporalBlurWeightPower = 1.0; + std::vector overlayLayers; }; constexpr int kMaxCursorAtlasEntries = 16; constexpr int kWebcamPrefetchOutputFrames = 900; +// Bounded overlay frame ring: slots are keyed by the clamped frame index so +// single-frame layers and tail-repeated frames are read from disk once and +// served from the device slot for every following output frame. Two slots of +// head room give a two-frame read-ahead without unbounded memory; the depth is +// always prefetchSlots - 2 so the ring never overwrites the slot the current +// blend is reading. +constexpr int kOverlayPrefetchSlots = 4; +constexpr int kOverlayPrefetchDepth = kOverlayPrefetchSlots - 2; [[noreturn]] void fail(const std::string& message) { throw std::runtime_error(message); } +const char* outputCodecName(OutputCodec codec) { + return codec == OutputCodec::HEVC ? "hevc" : "h264"; +} + +OutputCodec parseOutputCodec(const char* value) { + const std::string codec = value; + if (codec == "h264") { + return OutputCodec::H264; + } + if (codec == "hevc") { + return OutputCodec::HEVC; + } + fail("Unsupported --output-codec: " + codec + "; expected h264 or hevc"); +} + void checkCuda(cudaError_t status, const char* expression) { if (status != cudaSuccess) { std::ostringstream stream; @@ -167,6 +223,8 @@ Options parseOptions(int argc, char** argv) { options.inputPath = requireValue("--input"); } else if (arg == "--output") { options.outputPath = requireValue("--output"); + } else if (arg == "--output-codec") { + options.outputCodec = parseOutputCodec(requireValue("--output-codec")); } else if (arg == "--source-pts") { options.sourcePtsPath = requireValue("--source-pts"); } else if (arg == "--width") { @@ -278,9 +336,31 @@ Options parseOptions(int argc, char** argv) { parsePositiveInt(requireValue("--cursor-atlas-height"), "--cursor-atlas-height"); } else if (arg == "--zoom-samples") { options.zoomSamplesPath = requireValue("--zoom-samples"); + } else if (arg == "--temporal-blur-sample-count") { + options.temporalBlurSampleCount = + parsePositiveInt(requireValue("--temporal-blur-sample-count"), "--temporal-blur-sample-count"); + } else if (arg == "--temporal-blur-shutter-fraction") { + options.temporalBlurShutterFraction = parseFiniteDouble( + requireValue("--temporal-blur-shutter-fraction"), + "--temporal-blur-shutter-fraction"); + } else if (arg == "--temporal-blur-weight-power") { + options.temporalBlurWeightPower = parseFiniteDouble( + requireValue("--temporal-blur-weight-power"), + "--temporal-blur-weight-power"); + } else if (arg == "--overlay") { + OverlayLayerDescriptor layer; + layer.path = requireValue("--overlay"); + layer.x = parseNonNegativeInt(requireValue("--overlay"), "--overlay x"); + layer.y = parseNonNegativeInt(requireValue("--overlay"), "--overlay y"); + layer.width = parsePositiveInt(requireValue("--overlay"), "--overlay width"); + layer.height = parsePositiveInt(requireValue("--overlay"), "--overlay height"); + layer.frameCount = + parsePositiveInt(requireValue("--overlay"), "--overlay frameCount"); + options.overlayLayers.push_back(layer); } else if (arg == "--help") { std::cout << "Usage: recordly-nvidia-cuda-compositor --input input.annexb.h264 " - "[--output out.h264] [--source-pts source-pts.csv] [--width N --height N] [--fps 30] " + "[--output out.h264] [--output-codec h264|hevc] " + "[--source-pts source-pts.csv] [--width N --height N] [--fps 30] " "[--max-frames N] [--bitrate-mbps N] [--encoding-mode fast|balanced|quality] " "[--post-select] [--callback-encode] [--stream-sync] [--prewarm-ms N] [--chunk-mb N] " "[--content-x N --content-y N --content-width N --content-height N --radius N] " @@ -291,7 +371,10 @@ Options parseOptions(int argc, char** argv) { "[--cursor-samples cursor.tsv --cursor-height N] " "[--cursor-atlas-rgba cursor.rgba --cursor-atlas-metadata cursor.tsv " "--cursor-atlas-width N --cursor-atlas-height N] " - "[--zoom-samples zoom.csv]\n"; + "[--zoom-samples zoom.csv] " + "[--temporal-blur-sample-count N --temporal-blur-shutter-fraction F " + "--temporal-blur-weight-power P] " + "[--overlay overlay.rgba x y width height frameCount]...\n"; std::exit(0); } else { std::ostringstream stream; @@ -308,9 +391,39 @@ Options parseOptions(int argc, char** argv) { if (options.width > 0 && (options.width % 2 != 0 || options.height % 2 != 0)) { fail("--width and --height must be even numbers for NV12 encoding"); } + for (const auto& layer : options.overlayLayers) { + if (layer.width <= 0 || layer.height <= 0 || layer.frameCount <= 0) { + fail("Invalid --overlay layer dimensions: " + layer.path); + } + } + if (options.temporalBlurSampleCount > 0) { + if (options.temporalBlurSampleCount < 3 || options.temporalBlurSampleCount > 61) { + fail("Invalid --temporal-blur-sample-count: " + + std::to_string(options.temporalBlurSampleCount)); + } + if (!std::isfinite(options.temporalBlurShutterFraction) || + options.temporalBlurShutterFraction < 0.18 || + options.temporalBlurShutterFraction > 3.0) { + fail("Invalid --temporal-blur-shutter-fraction"); + } + } return options; } +// The overlay canvas bounds depend on the output dimensions. With explicit +// --width/--height they are known at parse time; without them the canvas is +// resolved from the decoded source, so the caller validates with the resolved +// dimensions before the first frame is encoded. +void validateOverlayBounds(const Options& options, int outputWidth, int outputHeight) { + for (const auto& layer : options.overlayLayers) { + if (layer.x < 0 || layer.y < 0 || + layer.x + layer.width > outputWidth || + layer.y + layer.height > outputHeight) { + fail("Overlay layer exceeds the output canvas: " + layer.path); + } + } +} + bool shouldEncodeFrame(int sourceFrameIndex, int encodedFrames, const Options& options) { if (options.inputFrames <= 0 || options.targetFrames <= 0) { return true; @@ -578,6 +691,10 @@ struct ProgressCounters { double decodeWallMs = 0.0; double encodeMs = 0.0; double compositeMs = 0.0; + double compositeGpuMs = 0.0; + double zoomBlurGpuMs = 0.0; + double overlayBlendGpuMs = 0.0; + double overlayUploadMs = 0.0; double nvencMs = 0.0; double packetWriteMs = 0.0; double webcamDecodeMs = 0.0; @@ -585,11 +702,28 @@ struct ProgressCounters { int roiCompositeFrames = 0; int monolithicCompositeFrames = 0; int copyCompositeFrames = 0; + int zoomBlurFrames = 0; + int overlayBlendFrames = 0; + int temporalBlurFrames = 0; + int64_t temporalBlurSamplesTotal = 0; + int temporalBlurBgPrecomposedFrames = 0; + int temporalBlurStationaryFrames = 0; + int temporalBgCacheBuilds = 0; + int64_t temporalBgCacheHits = 0; + int64_t overlayStaticRegionBlends = 0; + int64_t overlayFileLoads = 0; + int64_t overlayCacheHits = 0; + int64_t overlayPinnedHits = 0; + int64_t overlayReadWaits = 0; + int64_t overlayPendingReadsPeak = 0; + double overlayHostReadMs = 0.0; + double overlayH2DEnqueueMs = 0.0; }; struct ProgressReportState { std::chrono::steady_clock::time_point startedAt; std::chrono::steady_clock::time_point lastReportAt; + const char* outputCodec = "h264"; int lastReportedFrame = 0; ProgressCounters lastCounters; }; @@ -773,6 +907,14 @@ struct ZoomSample { double scale = 1.0; double x = 0.0; double y = 0.0; + // Renderer-equivalent radial zoom-blur parameters for the step that ends at + // this sample. blurStrength is the ZoomBlurFilter strength (0 = no blur); + // the center is in output pixels. The JS side computes these from the same + // camera-step analysis the interactive renderer uses, so the native + // compositor reproduces the spatial zoom blur without re-deriving it. + double blurStrength = 0.0; + double blurCenterX = 0.0; + double blurCenterY = 0.0; }; struct ZoomTrack { @@ -813,10 +955,71 @@ struct ZoomTrack { left.scale + (right.scale - left.scale) * t, left.x + (right.x - left.x) * t, left.y + (right.y - left.y) * t, + left.blurStrength + (right.blurStrength - left.blurStrength) * t, + left.blurCenterX + (right.blurCenterX - left.blurCenterX) * t, + left.blurCenterY + (right.blurCenterY - left.blurCenterY) * t, }; } }; +struct TemporalBlurSample { + double offsetUs = 0.0; + double weight = 0.0; +}; + +// Mirrors buildTemporalSamplePlanUs from src/lib/exporter/temporalMotionBlur.ts: +// symmetric shutter window centered on the frame, cos-tapered weights normalized +// to sum to 1. The weight floor (0.22) and taper are part of the renderer's +// contract; the compositor must reproduce them so native output matches the +// configured high-level temporal sample plan. +std::vector buildTemporalSamplePlan( + int sampleCount, + double shutterFraction, + double weightCurvePower, + double frameDurationUs) { + const int safeSampleCount = std::max(1, sampleCount); + if (safeSampleCount <= 1) { + return {{0.0, 1.0}}; + } + + const double shutterWindowUs = + std::max(1.0, frameDurationUs) * std::max(0.0, std::min(3.0, shutterFraction)); + const double startOffsetUs = -shutterWindowUs / 2.0; + const double stepUs = shutterWindowUs / static_cast(safeSampleCount - 1); + std::vector offsetsUs; + offsetsUs.reserve(safeSampleCount); + for (int index = 0; index < safeSampleCount; ++index) { + offsetsUs.push_back(startOffsetUs + stepUs * static_cast(index)); + } + + constexpr double kWeightFloor = 0.22; + const double centerIndex = static_cast(safeSampleCount - 1) / 2.0; + std::vector rawWeights; + rawWeights.reserve(safeSampleCount); + double totalWeight = 0.0; + for (int index = 0; index < safeSampleCount; ++index) { + const double normalizedDistance = + std::abs(static_cast(index) - centerIndex) / std::max(1.0, centerIndex); + const double taperedWeight = std::cos(normalizedDistance * (3.14159265358979323846 / 2.0)); + const double rawWeight = + kWeightFloor + + (1.0 - kWeightFloor) * + std::pow(std::max(0.0, taperedWeight), weightCurvePower); + rawWeights.push_back(rawWeight); + totalWeight += rawWeight; + } + + std::vector samples; + samples.reserve(safeSampleCount); + for (int index = 0; index < safeSampleCount; ++index) { + samples.push_back({ + offsetsUs[index], + totalWeight > 0.0 ? rawWeights[index] / totalWeight : 1.0 / safeSampleCount, + }); + } + return samples; +} + std::unique_ptr loadZoomTrack(const Options& options) { if (options.zoomSamplesPath.empty()) { return nullptr; @@ -843,8 +1046,26 @@ std::unique_ptr loadZoomTrack(const Options& options) { !std::isfinite(sample.x) || !std::isfinite(sample.y)) { continue; } + // Optional renderer-computed zoom-blur fields (columns 5-7). Older + // telemetry files with only timeMs/scale/x/y keep blurStrength = 0. + if (!(row >> sample.blurStrength)) { + sample.blurStrength = 0.0; + } else if (!std::isfinite(sample.blurStrength)) { + sample.blurStrength = 0.0; + } + if (!(row >> sample.blurCenterX)) { + sample.blurCenterX = 0.0; + } else if (!std::isfinite(sample.blurCenterX)) { + sample.blurCenterX = 0.0; + } + if (!(row >> sample.blurCenterY)) { + sample.blurCenterY = 0.0; + } else if (!std::isfinite(sample.blurCenterY)) { + sample.blurCenterY = 0.0; + } sample.timeMs = std::max(0.0, sample.timeMs); sample.scale = std::max(0.01, sample.scale); + sample.blurStrength = std::max(0.0, sample.blurStrength); track->samples.push_back(sample); } if (track->samples.empty()) { @@ -856,6 +1077,547 @@ std::unique_ptr loadZoomTrack(const Options& options) { return track; } +// Blend launch rectangle for a renderer-prepared RGBA overlay layer. Dynamic +// (multi-frame) layers always use the full layer rect; physical single-frame +// layers get a one-time alpha bound (see computeStaticAlphaBounds) so the blend +// kernel only visits pixels that can write, with the full rect as the fallback. +struct OverlayBlendRegion { + int x = 0; + int y = 0; + int width = 0; + int height = 0; + bool bounded = false; +}; + +OverlayBlendRegion fullOverlayBlendRegion(const OverlayLayerDescriptor& descriptor) { + OverlayBlendRegion region; + region.x = 0; + region.y = 0; + region.width = descriptor.width; + region.height = descriptor.height; + region.bounded = false; + return region; +} + +// Scans the first frame of a physical single-frame overlay layer for the +// bounding box of pixels with nonzero alpha, expanded by one pixel so every 2x2 +// chroma block that touches an alpha pixel is inside the launch region. The +// bound is computed once per layer; pixels outside it have alpha == 0 for the +// whole layer, so the blend kernel writes nothing there and the bounded launch +// is bit-identical to the full-frame blend. A fully transparent layer gets an +// empty region (the blend launch is skipped entirely, which is also exact). +void computeStaticAlphaBounds( + const unsigned char* rgba, + int width, + int height, + OverlayBlendRegion& region) { + int minX = width; + int minY = height; + int maxX = -1; + int maxY = -1; + for (int y = 0; y < height; ++y) { + const unsigned char* row = + rgba + static_cast(y) * static_cast(width) * 4; + for (int x = 0; x < width; ++x) { + if (row[x * 4 + 3] > 0) { + minX = std::min(minX, x); + minY = std::min(minY, y); + maxX = std::max(maxX, x); + maxY = std::max(maxY, y); + } + } + } + if (maxX < minX) { + region = {0, 0, 0, 0, true}; + return; + } + region.x = std::max(0, minX - 1); + region.y = std::max(0, minY - 1); + region.width = std::min(width, maxX + 2) - region.x; + region.height = std::min(height, maxY + 2) - region.y; + region.bounded = true; +} + +// Streaming source for renderer-prepared transparent RGBA overlay sidecars. +// Frames are raw top-down RGBA and are consumed sequentially by the output +// frame index. Each dynamic layer owns a bounded background reader thread that +// reads sidecar frames from disk into persistent pinned ring buffers while the +// encode thread keeps running, so the encode loop never blocks on file I/O. +// The encode thread only enqueues H2D copies (pinned -> device) on the +// compositor stream, ordered ahead of the blend kernels on the same stream, so +// the single per-frame cudaStreamSynchronize stays sufficient. The 4-slot ring +// semantics are unchanged: frame ordering, tail-repeat clamping, the read-once +// static cache, and bounded memory are all preserved. +class OverlayFrameSource { +public: + explicit OverlayFrameSource(const std::vector& layers) { + layers_.reserve(layers.size()); + for (const auto& descriptor : layers) { + std::unique_ptr layer = loadLayer(descriptor); + if (layer->staticLayer) { + // The constructor read of the static layer's single frame is a + // disk read with an upload (synchronous), so it counts as a + // file load like the streaming path counts its reads. + ++fileLoads_; + } + layers_.push_back(std::move(layer)); + } + // Start one reader thread per dynamic layer only after every layer is + // fully built (vector is stable and static frames are staged), so a + // reader can never observe a partially initialized layer. + for (auto& layer : layers_) { + if (!layer->staticLayer) { + startReader(*layer); + } + } + } + + ~OverlayFrameSource() { + // Stop and join every reader before freeing the pinned buffers the + // readers write into. Reader threads never touch CUDA, so joining is + // safe while the primary context is current. + for (auto& layer : layers_) { + stopReader(*layer); + } + for (auto& layer : layers_) { + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + if (layer->deviceFrames[slot]) { + cudaFree(layer->deviceFrames[slot]); + } + if (layer->pinnedFrames[slot]) { + cudaFreeHost(layer->pinnedFrames[slot]); + } + } + } + } + + bool empty() const { + return layers_.empty(); + } + + size_t layerCount() const { + return layers_.size(); + } + + // Total streaming-path overlay time (background host reads + H2D + // enqueues). Static constructor staging is intentionally excluded, matching + // the pre-background-reader semantics of uploadMs. + double uploadMs() const { + return hostReadMs() + h2dEnqueueMs(); + } + + // Wall time the background reader threads spent reading sidecar bytes from + // disk (not the H2D transfer time). + double hostReadMs() const { + return static_cast(hostReadUs_.load()) / 1000.0; + } + + // Wall time the encode thread spent enqueuing H2D copies (cudaMemcpyAsync + // API calls) on the compositor stream. + double h2dEnqueueMs() const { + return h2dEnqueueMs_; + } + + // Number of overlay frames read from disk (one per unique requested frame; + // static single-frame layers count their one constructor read). + int64_t fileLoads() const { + return fileLoads_.load(); + } + + // Number of times a requested overlay frame was already device-resident in + // its ring slot, so neither a file read nor an H2D copy was needed. Static + // single-frame layers, tail-repeated frames, and read-ahead frames all + // count here. + int64_t cacheHits() const { + return cacheHits_; + } + + // Number of times a requested overlay frame was already in pinned memory + // (the background reader had finished the file read) and only the H2D + // enqueue was needed. + int64_t pinnedHits() const { + return pinnedHits_; + } + + // Number of times the encode thread had to wait for the background reader + // to finish a file read before it could enqueue the H2D copy. + int64_t readWaits() const { + return readWaits_; + } + + // Peak depth of the bounded background-reader queue across all layers. + int64_t pendingReadsPeak() const { + return pendingReadsPeak_.load(); + } + + const OverlayLayerDescriptor& descriptor(size_t index) const { + return layers_[index]->descriptor; + } + + // Blend launch rectangle for the layer. Dynamic (multi-frame) layers return + // the full layer rect; physical single-frame layers return the one-time + // alpha bound (or an empty rect for a fully transparent layer). + OverlayBlendRegion blendRegion(size_t index) const { + return layers_[index]->blendRegion; + } + + // Prepares the overlay frame for the given output frame index for every + // layer. Call this before launching the blend kernels: it waits (host-side) + // only when the background reader has not finished the requested frame, + // then enqueues the H2D copy on the compositor stream so blends stay + // ordered. Slots are keyed by the clamped frame index inside a small + // bounded ring, so a single-frame layer or a tail-repeated last frame is + // read from disk once and served from its device slot for every following + // output frame. This never syncs the compositor stream; the encode loop + // keeps its single per-frame cudaStreamSynchronize. + void beginFrame(int outputFrameIndex, cudaStream_t copyStream) { + if (layers_.empty()) { + return; + } + + for (size_t index = 0; index < layers_.size(); ++index) { + auto& layer = *layers_[index]; + const int frameIndex = clampedFrameIndex(layer, outputFrameIndex); + const int slot = slotFor(frameIndex); + if (layer.staticLayer) { + // Static layers are fully staged in the constructor; slot 0 is + // always device-resident for frame 0. + ++cacheHits_; + continue; + } + waitForFrame(layer, frameIndex, slot, copyStream); + } + } + + const unsigned char* frameDevicePtr(size_t layerIndex, int outputFrameIndex) const { + const auto& layer = *layers_[layerIndex]; + return layer.deviceFrames[slotFor(clampedFrameIndex(layer, outputFrameIndex))]; + } + + // Must be called after beginFrame + the blend kernels are queued (after the + // per-frame stream sync). Dispatches bounded background reads for the next + // overlay frames so the following output frames do not stall on file I/O; + // when a read-ahead frame's pinned data is already available it also + // enqueues the H2D copy immediately so the transfer overlaps NVENC. + // Read-ahead depth is bounded by the ring size minus one and never targets + // the slot the current blend is reading, so the pipeline stays ordered with + // bounded memory. + void prefetchNextFrame(int outputFrameIndex, cudaStream_t copyStream) { + if (layers_.empty()) { + return; + } + + for (size_t index = 0; index < layers_.size(); ++index) { + auto& layer = *layers_[index]; + if (layer.staticLayer) { + continue; + } + const int currentFrameIndex = clampedFrameIndex(layer, outputFrameIndex); + const int currentSlot = slotFor(currentFrameIndex); + for (int depth = 1; depth <= kOverlayPrefetchDepth; ++depth) { + const int frameIndex = clampedFrameIndex(layer, outputFrameIndex + depth); + if (frameIndex == currentFrameIndex || slotFor(frameIndex) == currentSlot) { + continue; + } + requestRead(layer, frameIndex, slotFor(frameIndex), copyStream); + } + } + } + +private: + enum class SlotState { + Empty, + Reading, + PinnedReady, + DeviceReady, + }; + + struct LoadedLayer { + OverlayLayerDescriptor descriptor; + size_t frameBytes = 0; + std::ifstream input; + unsigned char* deviceFrames[kOverlayPrefetchSlots] = {}; + unsigned char* pinnedFrames[kOverlayPrefetchSlots] = {}; + int loadedSlots[kOverlayPrefetchSlots] = {}; + SlotState slotStates[kOverlayPrefetchSlots] = {}; + OverlayBlendRegion blendRegion; + bool staticLayer = false; + // Bounded background reader state (dynamic layers only). The pending + // queue never holds more than one entry per ring slot, so it is bounded + // by kOverlayPrefetchSlots; the reader thread is the only accessor of + // input and pinnedFrames outside the constructor. + std::mutex mutex; + std::condition_variable cv; + std::deque> pendingReads; + bool stop = false; + bool readerStarted = false; + std::thread readerThread; + std::string readError; + }; + + static int clampedFrameIndex(const LoadedLayer& layer, int outputFrameIndex) { + return std::min(outputFrameIndex, std::max(0, layer.descriptor.frameCount - 1)); + } + + static int slotFor(int frameIndex) { + return frameIndex % kOverlayPrefetchSlots; + } + + static std::unique_ptr loadLayer(const OverlayLayerDescriptor& descriptor) { + std::unique_ptr layer = std::make_unique(); + layer->descriptor = descriptor; + layer->frameBytes = static_cast(descriptor.width) * + static_cast(descriptor.height) * 4; + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + layer->loadedSlots[slot] = -1; + layer->slotStates[slot] = SlotState::Empty; + } + + layer->input.open(descriptor.path, std::ios::binary); + if (!layer->input) { + fail("Failed to open overlay layer: " + descriptor.path); + } + layer->input.seekg(0, std::ios::end); + const std::streampos end = layer->input.tellg(); + layer->input.seekg(0, std::ios::beg); + if (end < 0 || + static_cast(end) < layer->frameBytes * static_cast(descriptor.frameCount)) { + fail("Overlay layer is truncated: " + descriptor.path); + } + + for (int slot = 0; slot < kOverlayPrefetchSlots; ++slot) { + checkCuda(cudaMalloc(&layer->deviceFrames[slot], layer->frameBytes), "cudaMalloc overlay frame"); + checkCuda(cudaMallocHost(&layer->pinnedFrames[slot], layer->frameBytes), "cudaMallocHost overlay frame"); + } + + // Physical single-frame layers are invariant for the whole export: read + // the single frame once, compute the alpha bounds once, and stage the + // device copy now so beginFrame serves it from slot 0 without a second + // file read. The ring is keyed by the clamped frame index, which is + // always 0 for a frameCount == 1 layer, so slot 0 stays valid forever. + layer->staticLayer = descriptor.frameCount == 1; + layer->blendRegion = fullOverlayBlendRegion(descriptor); + if (layer->staticLayer) { + layer->input.seekg(0, std::ios::beg); + layer->input.read( + reinterpret_cast(layer->pinnedFrames[0]), + static_cast(layer->frameBytes)); + if (static_cast(layer->input.gcount()) != layer->frameBytes) { + fail("Failed to read overlay frame 0: " + descriptor.path); + } + computeStaticAlphaBounds( + layer->pinnedFrames[0], + descriptor.width, + descriptor.height, + layer->blendRegion); + // Static staging (read + upload) is intentionally not timed: it is + // a one-time constructor cost and the streaming-path timing metrics + // (hostReadMs/h2dEnqueueMs) exclude it, matching the historical + // uploadMs semantics. + checkCuda( + cudaMemcpy( + layer->deviceFrames[0], + layer->pinnedFrames[0], + layer->frameBytes, + cudaMemcpyHostToDevice), + "cudaMemcpy overlay static frame 0"); + layer->loadedSlots[0] = 0; + layer->slotStates[0] = SlotState::DeviceReady; + } + return layer; + } + + void startReader(LoadedLayer& layer) { + std::unique_lock lock(layer.mutex); + layer.readerStarted = true; + layer.readerThread = std::thread(&OverlayFrameSource::readerLoop, this, &layer); + } + + void stopReader(LoadedLayer& layer) { + { + std::unique_lock lock(layer.mutex); + layer.stop = true; + } + layer.cv.notify_all(); + if (layer.readerStarted && layer.readerThread.joinable()) { + layer.readerThread.join(); + } + } + + // Background reader main loop: pops the oldest queued (slot, frameIndex) + // read, performs the file read into the persistent pinned buffer, and + // publishes the PinnedReady state. The queue is bounded (one entry per ring + // slot) and the loop never touches CUDA, so cancellation is a simple stop + // flag + join; a read failure is captured and re-thrown on the encode + // thread at the next beginFrame. + void readerLoop(LoadedLayer* layer) { + while (true) { + std::pair request; + { + std::unique_lock lock(layer->mutex); + layer->cv.wait(lock, [&] { + return layer->stop || !layer->pendingReads.empty(); + }); + if (layer->stop) { + return; + } + request = layer->pendingReads.front(); + layer->pendingReads.pop_front(); + } + readFrameIntoPinned(*layer, request.first, request.second); + } + } + + void readFrameIntoPinned(LoadedLayer& layer, int slot, int frameIndex) { + const auto readStart = std::chrono::steady_clock::now(); + try { + layer.input.seekg( + static_cast(layer.frameBytes * static_cast(frameIndex)), + std::ios::beg); + layer.input.read( + reinterpret_cast(layer.pinnedFrames[slot]), + static_cast(layer.frameBytes)); + if (static_cast(layer.input.gcount()) != layer.frameBytes) { + throw std::runtime_error( + "Failed to read overlay frame " + std::to_string(frameIndex) + ": " + + layer.descriptor.path); + } + } catch (const std::exception& error) { + std::unique_lock lock(layer.mutex); + layer.readError = error.what(); + layer.stop = true; + layer.cv.notify_all(); + return; + } + hostReadUs_ += static_cast(elapsedMs(readStart, std::chrono::steady_clock::now()) * 1000.0); + ++fileLoads_; + { + std::unique_lock lock(layer.mutex); + layer.loadedSlots[slot] = frameIndex; + layer.slotStates[slot] = SlotState::PinnedReady; + layer.cv.notify_all(); + } + } + + // Queues a background read for (slot, frameIndex) unless one is already in + // flight/queued for that slot. A newer request supersedes a stale queued + // target for the same slot (the older frame's blend already consumed its + // device data, so overwriting the pinned buffer is safe). The queue is + // bounded to one entry per ring slot; if it is full the request is dropped + // and the caller's wait loop retries once the reader drains an entry. + // Must be called with layer.mutex held. + void requestReadLocked(LoadedLayer& layer, int frameIndex, int slot) { + if (layer.slotStates[slot] == SlotState::Reading && + layer.loadedSlots[slot] == frameIndex) { + return; + } + for (auto& entry : layer.pendingReads) { + if (entry.first == slot) { + if (entry.second != frameIndex) { + entry.second = frameIndex; + } + layer.cv.notify_one(); + return; + } + } + if (layer.pendingReads.size() >= static_cast(kOverlayPrefetchSlots)) { + return; + } + layer.pendingReads.push_back({slot, frameIndex}); + pendingReadsPeak_.store( + std::max(pendingReadsPeak_.load(), static_cast(layer.pendingReads.size()))); + layer.slotStates[slot] = SlotState::Reading; + layer.loadedSlots[slot] = frameIndex; + layer.cv.notify_one(); + } + + // Non-blocking read-ahead request (prefetch path): queues the background + // read and, when the pinned data is already available, enqueues the H2D + // copy immediately so it overlaps NVENC instead of the next beginFrame. + void requestRead(LoadedLayer& layer, int frameIndex, int slot, cudaStream_t copyStream) { + std::unique_lock lock(layer.mutex); + if (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex) { + ++cacheHits_; + return; + } + if (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) { + ++pinnedHits_; + lock.unlock(); + enqueueH2D(layer, slot, copyStream); + lock.lock(); + layer.slotStates[slot] = SlotState::DeviceReady; + return; + } + requestReadLocked(layer, frameIndex, slot); + } + + // Ensures the requested overlay frame's pinned data is available and its + // H2D copy is enqueued on the compositor stream. Waits on the background + // reader are host-side (condition variable) and never sync the stream; the + // encode loop keeps its single per-frame cudaStreamSynchronize. + void waitForFrame(LoadedLayer& layer, int frameIndex, int slot, cudaStream_t copyStream) { + std::unique_lock lock(layer.mutex); + while (true) { + if (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex) { + ++cacheHits_; + return; + } + if (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) { + ++pinnedHits_; + lock.unlock(); + enqueueH2D(layer, slot, copyStream); + lock.lock(); + layer.slotStates[slot] = SlotState::DeviceReady; + return; + } + if (!layer.readError.empty()) { + fail(layer.readError); + } + if (layer.stop) { + fail("Overlay reader stopped before frame " + std::to_string(frameIndex)); + } + requestReadLocked(layer, frameIndex, slot); + ++readWaits_; + layer.cv.wait(lock, [&] { + return layer.stop || !layer.readError.empty() || + (layer.slotStates[slot] == SlotState::PinnedReady && + layer.loadedSlots[slot] == frameIndex) || + (layer.slotStates[slot] == SlotState::DeviceReady && + layer.loadedSlots[slot] == frameIndex); + }); + } + } + + // Enqueues the H2D copy for a PinnedReady slot on the compositor stream. + // Main thread only; the transfer is ordered ahead of the blend kernels on + // the same stream. + void enqueueH2D(LoadedLayer& layer, int slot, cudaStream_t copyStream) { + const auto enqueueStart = std::chrono::steady_clock::now(); + checkCuda( + cudaMemcpyAsync( + layer.deviceFrames[slot], + layer.pinnedFrames[slot], + layer.frameBytes, + cudaMemcpyHostToDevice, + copyStream), + "cudaMemcpyAsync overlay frame"); + h2dEnqueueMs_ += elapsedMs(enqueueStart, std::chrono::steady_clock::now()); + } + + std::vector> layers_; + std::atomic fileLoads_{0}; + std::atomic hostReadUs_{0}; + std::atomic pendingReadsPeak_{0}; + double h2dEnqueueMs_ = 0.0; + int64_t cacheHits_ = 0; + int64_t pinnedHits_ = 0; + int64_t readWaits_ = 0; +}; + struct CursorAtlasEntry { int x = 0; int y = 0; @@ -1484,6 +2246,84 @@ __device__ int sampleCursorAtlasShadowAlpha( return min(255, weightedAlpha / 100); } +__device__ __forceinline__ unsigned char temporalAccumulateByte( + unsigned char current, + unsigned char value, + unsigned int weightFixed, + int accumulateMode) { + if (accumulateMode == 0) { + // Legacy direct write (non-temporal composites). + return value; + } + const int weighted = (static_cast(weightFixed) * static_cast(value) + 128) >> 8; + if (accumulateMode == 1) { + // First temporal sample: replace (the target is not pre-zeroed, so this + // must not read stale buffer contents). Matches the previous + // zero-fill + (weight * value + 128) >> 8 accumulate exactly. + return static_cast(min(255, weighted)); + } + // Subsequent temporal samples: saturating accumulate into the target. + return static_cast(min(255, static_cast(current) + weighted)); +} + +// Accumulates the temporal sample weights applied to the invariant background +// into one full-frame pass. Every pixel outside the transformed content +// bounding box maps outside the content rect for every temporal sample, so its +// per-sample composite value is always the background; the saturating weighted +// sum of the background is therefore identical for all samples and can be +// computed once per output frame. The term-for-term math reproduces the +// replace-then-saturate-accumulate chain of compositeStaticNv12Kernel exactly +// (same (weight * value + 128) >> 8 per sample, same saturation), including +// per-sample rounding, so pixels served by this pass are bit-identical to the +// previous per-sample full-frame composites. +__global__ void accumulateBackgroundNv12Kernel( + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + unsigned char backgroundY, + unsigned char backgroundU, + unsigned char backgroundV, + const unsigned char* background, + const unsigned int* sampleWeights, + int sampleCount) { + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x >= dstWidth || y >= dstHeight || sampleCount <= 0) { + return; + } + + const unsigned int bgY = background ? background[y * dstWidth + x] : backgroundY; + unsigned int yAcc = (sampleWeights[0] * bgY + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int term = (sampleWeights[index] * bgY + 128u) >> 8; + yAcc = min(255u, yAcc + term); + } + dst[y * dstPitch + x] = static_cast(yAcc); + + if ((x % 2) == 0 && (y % 2) == 0) { + unsigned int bgU = backgroundU; + unsigned int bgV = backgroundV; + if (background) { + const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; + bgU = bgUv[0]; + bgV = bgUv[1]; + } + unsigned int uAcc = (sampleWeights[0] * bgU + 128u) >> 8; + unsigned int vAcc = (sampleWeights[0] * bgV + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int uTerm = (sampleWeights[index] * bgU + 128u) >> 8; + const unsigned int vTerm = (sampleWeights[index] * bgV + 128u) >> 8; + uAcc = min(255u, uAcc + uTerm); + vAcc = min(255u, vAcc + vTerm); + } + unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + dstUv[0] = static_cast(uAcc); + dstUv[1] = static_cast(vAcc); + } +} + __global__ void compositeStaticNv12Kernel( const unsigned char* src, int srcPitch, @@ -1495,6 +2335,10 @@ __global__ void compositeStaticNv12Kernel( int dstChromaOffset, int dstWidth, int dstHeight, + int regionX, + int regionY, + int regionWidth, + int regionHeight, int contentX, int contentY, int contentWidth, @@ -1533,10 +2377,17 @@ __global__ void compositeStaticNv12Kernel( bool zoomEnabled, float zoomScale, float zoomX, - float zoomY) { - const int x = blockIdx.x * blockDim.x + threadIdx.x; - const int y = blockIdx.y * blockDim.y + threadIdx.y; - if (x >= dstWidth || y >= dstHeight) { + float zoomY, + unsigned int temporalWeightFixed, + int temporalAccumulateMode) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight) { + return; + } + const int x = regionX + localX; + const int y = regionY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { return; } @@ -1638,10 +2489,16 @@ __global__ void compositeStaticNv12Kernel( outY = 16; } } - dst[y * dstPitch + x] = outY; + dst[y * dstPitch + x] = temporalAccumulateByte( + dst[y * dstPitch + x], + outY, + temporalWeightFixed, + temporalAccumulateMode); if ((x % 2) == 0 && (y % 2) == 0) { unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + unsigned char outU = backgroundU; + unsigned char outV = backgroundV; const float uvLayoutXf = zoomActive ? (static_cast(x + 1) - zoomX) / safeZoomScale : static_cast(x + 1); const float uvLayoutYf = @@ -1664,16 +2521,16 @@ __global__ void compositeStaticNv12Kernel( const int suvY = min((srcHeight / 2) - 1, (cropY + static_cast(localY * cropHeight / contentHeight)) / 2); const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX; - dstUv[0] = srcUv[0]; - dstUv[1] = srcUv[1]; + outU = srcUv[0]; + outV = srcUv[1]; } else { if (background) { const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; - dstUv[0] = bgUv[0]; - dstUv[1] = bgUv[1]; + outU = bgUv[0]; + outV = bgUv[1]; } else { - dstUv[0] = backgroundU; - dstUv[1] = backgroundV; + outU = backgroundU; + outV = backgroundV; } } if (webcam && @@ -1694,8 +2551,8 @@ __global__ void compositeStaticNv12Kernel( const int webcamUvY = min((webcamFrameHeight / 2) - 1, sampleY / 2); const unsigned char* webcamUv = webcam + webcamFrameWidth * webcamFrameHeight + webcamUvY * webcamFrameWidth + webcamUvX; - dstUv[0] = webcamUv[0]; - dstUv[1] = webcamUv[1]; + outU = webcamUv[0]; + outV = webcamUv[1]; } unsigned char cursorUvY = 0; unsigned char cursorUvU = 128; @@ -1718,8 +2575,8 @@ __global__ void compositeStaticNv12Kernel( y + 1) : 0; if (cursorUvShadowAlpha > 0) { - dstUv[0] = blendByte(dstUv[0], 128, cursorUvShadowAlpha); - dstUv[1] = blendByte(dstUv[1], 128, cursorUvShadowAlpha); + outU = blendByte(outU, 128, cursorUvShadowAlpha); + outV = blendByte(outV, 128, cursorUvShadowAlpha); } const bool cursorAtlasUvHit = cursorVisible && @@ -1742,51 +2599,214 @@ __global__ void compositeStaticNv12Kernel( &cursorUvV, &cursorUvAlpha); if (cursorAtlasUvHit) { - dstUv[0] = blendByte(dstUv[0], cursorUvU, cursorUvAlpha); - dstUv[1] = blendByte(dstUv[1], cursorUvV, cursorUvAlpha); + outU = blendByte(outU, cursorUvU, cursorUvAlpha); + outV = blendByte(outV, cursorUvV, cursorUvAlpha); } else { const int cursorUvMask = cursorVisible && !cursorAtlasRgba ? cursorMaskAt(x + 1, y + 1, cursorX, cursorY, cursorWidth, cursorHeight) : 0; if (cursorUvMask > 0) { - dstUv[0] = 128; - dstUv[1] = 128; + outU = 128; + outV = 128; } } + dstUv[0] = temporalAccumulateByte( + dstUv[0], + outU, + temporalWeightFixed, + temporalAccumulateMode); + dstUv[1] = temporalAccumulateByte( + dstUv[1], + outV, + temporalWeightFixed, + temporalAccumulateMode); } } -__global__ void overlayWebcamNv12Kernel( +// Fused constant-transform temporal composition: evaluates the source + layout +// composite value exactly once per pixel (the same content/bg/shadow selection +// compositeStaticNv12Kernel makes for the temporal path, where webcam/cursor +// are applied afterward) and then applies the existing fixed-point weights in +// order: sample 0 replaces with (w0 * v + 128) >> 8 and later samples +// saturate-accumulate (w * v + 128) >> 8. This is only launched when the +// stationary shutter-window check proved every sample resolves to the same +// camera transform (bit-identical scale/x/y), so the per-sample composite value +// is identical for every sample and the term-for-term math reproduces the +// per-sample replace-then-accumulate chain exactly, including per-sample +// rounding and progressive saturation. +__global__ void compositeStaticStationaryNv12Kernel( + const unsigned char* src, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, unsigned char* dst, int dstPitch, int dstChromaOffset, int dstWidth, int dstHeight, - const unsigned char* webcam, int regionX, int regionY, int regionWidth, int regionHeight, - int webcamX, - int webcamY, - int webcamSize, - int webcamFrameWidth, - int webcamFrameHeight, - int webcamRadius, - bool webcamMirror) { - const int localX = blockIdx.x * blockDim.x + threadIdx.x; - const int localY = blockIdx.y * blockDim.y + threadIdx.y; - if (!webcam || localX >= regionWidth || localY >= regionHeight) { - return; - } - - const int x = regionX + localX; - const int y = regionY + localY; - if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { - return; - } - + int contentX, + int contentY, + int contentWidth, + int contentHeight, + int sourceCropX, + int sourceCropY, + int sourceCropWidth, + int sourceCropHeight, + int radius, + unsigned char backgroundY, + unsigned char backgroundU, + unsigned char backgroundV, + const unsigned char* background, + int shadowOffsetY, + int shadowIntensityPct, + bool zoomEnabled, + float zoomScale, + float zoomX, + float zoomY, + const unsigned int* sampleWeights, + int sampleCount) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight || sampleCount <= 0) { + return; + } + const int x = regionX + localX; + const int y = regionY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + + const bool zoomActive = zoomEnabled && zoomScale > 0.01f; + const float safeZoomScale = fmaxf(zoomScale, 0.01f); + const float layoutXf = + zoomActive ? (static_cast(x) - zoomX) / safeZoomScale : static_cast(x); + const float layoutYf = + zoomActive ? (static_cast(y) - zoomY) / safeZoomScale : static_cast(y); + const int layoutX = static_cast(floorf(layoutXf)); + const int layoutY = static_cast(floorf(layoutYf)); + + const int cropX = max(0, min(sourceCropX, srcWidth - 1)); + const int cropY = max(0, min(sourceCropY, srcHeight - 1)); + const int cropWidth = max(1, min(sourceCropWidth > 0 ? sourceCropWidth : srcWidth, srcWidth - cropX)); + const int cropHeight = max(1, min(sourceCropHeight > 0 ? sourceCropHeight : srcHeight, srcHeight - cropY)); + const bool inside = + isInsideRoundedRect(layoutX, layoutY, contentX, contentY, contentWidth, contentHeight, radius); + unsigned char outY = background ? background[y * dstWidth + x] : backgroundY; + if (inside) { + const float localX = + fminf(static_cast(contentWidth - 1), fmaxf(0.0f, layoutXf - contentX)); + const float localY = + fminf(static_cast(contentHeight - 1), fmaxf(0.0f, layoutYf - contentY)); + const int sx = min(srcWidth - 1, cropX + static_cast((localX * cropWidth) / contentWidth)); + const int sy = min(srcHeight - 1, cropY + static_cast((localY * cropHeight) / contentHeight)); + outY = src[sy * srcPitch + sx]; + } else { + const bool shadowInside = + shadowIntensityPct > 0 && + isInsideRoundedRect( + layoutX, + layoutY, + contentX, + contentY + shadowOffsetY, + contentWidth, + contentHeight, + radius + 8); + if (shadowInside) { + const int darkenPct = min(75, max(0, shadowIntensityPct / 2)); + outY = static_cast((static_cast(outY) * (100 - darkenPct)) / 100); + } + } + unsigned int yAcc = (sampleWeights[0] * outY + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int term = (sampleWeights[index] * outY + 128u) >> 8; + yAcc = min(255u, yAcc + term); + } + dst[y * dstPitch + x] = static_cast(yAcc); + + if ((x % 2) == 0 && (y % 2) == 0) { + unsigned char* dstUv = dst + dstChromaOffset + (y / 2) * dstPitch + x; + unsigned char outU = backgroundU; + unsigned char outV = backgroundV; + const float uvLayoutXf = + zoomActive ? (static_cast(x + 1) - zoomX) / safeZoomScale : static_cast(x + 1); + const float uvLayoutYf = + zoomActive ? (static_cast(y + 1) - zoomY) / safeZoomScale : static_cast(y + 1); + const int uvLayoutX = static_cast(floorf(uvLayoutXf)); + const int uvLayoutY = static_cast(floorf(uvLayoutYf)); + const bool uvInside = isInsideRoundedRect( + uvLayoutX, + uvLayoutY, + contentX, + contentY, + contentWidth, + contentHeight, + radius); + if (uvInside) { + const float localX = + fminf(static_cast(contentWidth - 1), fmaxf(0.0f, uvLayoutXf - contentX)); + const float localY = + fminf(static_cast(contentHeight - 1), fmaxf(0.0f, uvLayoutYf - contentY)); + const int suvX = + min(srcWidth - 2, (cropX + static_cast((localX * cropWidth) / contentWidth)) & ~1); + const int suvY = + min((srcHeight / 2) - 1, (cropY + static_cast(localY * cropHeight / contentHeight)) / 2); + const unsigned char* srcUv = src + srcPitch * srcSurfaceHeight + suvY * srcPitch + suvX; + outU = srcUv[0]; + outV = srcUv[1]; + } else if (background) { + const unsigned char* bgUv = background + dstWidth * dstHeight + (y / 2) * dstWidth + x; + outU = bgUv[0]; + outV = bgUv[1]; + } + unsigned int uAcc = (sampleWeights[0] * outU + 128u) >> 8; + unsigned int vAcc = (sampleWeights[0] * outV + 128u) >> 8; + for (int index = 1; index < sampleCount; ++index) { + const unsigned int uTerm = (sampleWeights[index] * outU + 128u) >> 8; + const unsigned int vTerm = (sampleWeights[index] * outV + 128u) >> 8; + uAcc = min(255u, uAcc + uTerm); + vAcc = min(255u, vAcc + vTerm); + } + dstUv[0] = static_cast(uAcc); + dstUv[1] = static_cast(vAcc); + } +} + +__global__ void overlayWebcamNv12Kernel( + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + const unsigned char* webcam, + int regionX, + int regionY, + int regionWidth, + int regionHeight, + int webcamX, + int webcamY, + int webcamSize, + int webcamFrameWidth, + int webcamFrameHeight, + int webcamRadius, + bool webcamMirror) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (!webcam || localX >= regionWidth || localY >= regionHeight) { + return; + } + + const int x = regionX + localX; + const int y = regionY + localY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + if (isInsideRoundedRect(x, y, webcamX, webcamY, webcamSize, webcamSize, webcamRadius)) { const int webcamLocalX = max(0, min(webcamSize - 1, x - webcamX)); const int webcamLocalY = max(0, min(webcamSize - 1, y - webcamY)); @@ -1963,6 +2983,216 @@ __global__ void overlayCursorNv12Kernel( } } +__device__ float zoomBlurHash01(int x, int y) { + unsigned int value = static_cast(x) * 747796405u + + static_cast(y) * 2891336453u + 0x9e3779b9u; + value = value * 1664525u + 1013904223u; + value ^= value >> 13; + return static_cast(value & 0x00ffffffu) / 16777216.0f; +} + +// Spatial radial zoom blur equivalent to the renderer's ZoomBlurFilter applied +// to the transformed content. For each pixel the ray toward the blur center is +// sampled with a tent weight profile (4*(p-p^2)) over a fixed sample count, +// matching the pixi-filters zoom-blur shader with innerRadius=0/radius=-1 that +// the interactive renderer configures. Blur is restricted to the content region +// so webcam/cursor/background stay sharp, like the renderer's camera container. +// NV12 chroma is blurred at half resolution with the same radial ray. +__global__ void zoomBlurNv12Kernel( + const unsigned char* src, + int srcPitch, + int srcChromaOffset, + int dstWidth, + int dstHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int regionLeft, + int regionTop, + int regionRight, + int regionBottom, + float centerX, + float centerY, + float strength) { + constexpr int kZoomBlurSamples = 13; + const int x = blockIdx.x * blockDim.x + threadIdx.x; + const int y = blockIdx.y * blockDim.y + threadIdx.y; + if (x < regionLeft || x >= regionRight || y < regionTop || y >= regionBottom || + x >= dstWidth || y >= dstHeight) { + return; + } + + const float dirX = centerX - static_cast(x); + const float dirY = centerY - static_cast(y); + const float offset = zoomBlurHash01(x, y); + float total = 0.0f; + float acc = 0.0f; + for (int t = 0; t < kZoomBlurSamples; ++t) { + const float percent = (static_cast(t) + offset) / static_cast(kZoomBlurSamples); + const float weight = 4.0f * (percent - percent * percent); + const int sx = static_cast(static_cast(x) + dirX * strength * percent); + const int sy = static_cast(static_cast(y) + dirY * strength * percent); + const int clampedX = min(regionRight - 1, max(regionLeft, sx)); + const int clampedY = min(regionBottom - 1, max(regionTop, sy)); + acc += weight * static_cast(src[clampedY * srcPitch + clampedX]); + total += weight; + } + dst[y * dstPitch + x] = static_cast(acc / total + 0.5f); + + if ((x % 2) == 0 && (y % 2) == 0) { + const int ux = x / 2; + const int uy = y / 2; + const int uLeft = regionLeft / 2; + const int uTop = regionTop / 2; + const int uRight = min(dstWidth / 2, (regionRight + 1) / 2); + const int uBottom = min(dstHeight / 2, (regionBottom + 1) / 2); + if (ux >= uLeft && ux < uRight && uy >= uTop && uy < uBottom) { + const float uCenterX = centerX * 0.5f; + const float uCenterY = centerY * 0.5f; + const float uDirX = uCenterX - static_cast(ux); + const float uDirY = uCenterY - static_cast(uy); + const float uOffset = zoomBlurHash01(ux, uy); + float uTotal = 0.0f; + float uAcc = 0.0f; + float vAcc = 0.0f; + for (int t = 0; t < kZoomBlurSamples; ++t) { + const float percent = (static_cast(t) + uOffset) / static_cast(kZoomBlurSamples); + const float weight = 4.0f * (percent - percent * percent); + const int sux = min(uRight - 1, max(uLeft, static_cast(static_cast(ux) + uDirX * strength * percent))); + const int suy = min(uBottom - 1, max(uTop, static_cast(static_cast(uy) + uDirY * strength * percent))); + const unsigned char* uv = src + srcChromaOffset + suy * srcPitch + sux * 2; + uAcc += weight * static_cast(uv[0]); + vAcc += weight * static_cast(uv[1]); + uTotal += weight; + } + unsigned char* dstUv = dst + dstChromaOffset + uy * dstPitch + ux * 2; + dstUv[0] = static_cast(uAcc / uTotal + 0.5f); + dstUv[1] = static_cast(vAcc / uTotal + 0.5f); + } + } +} + +__device__ void rgbaToNv12Yuv(int r, int g, int b, unsigned char& y, unsigned char& u, unsigned char& v) { + y = clampByteDevice(((66 * r + 129 * g + 25 * b + 128) >> 8) + 16); + u = clampByteDevice(((-38 * r - 74 * g + 112 * b + 128) >> 8) + 128); + v = clampByteDevice(((112 * r - 94 * g - 18 * b + 128) >> 8) + 128); +} + +// Blends a transparent top-down RGBA overlay layer over the composed NV12 +// frame. Luma is blended per pixel; chroma is averaged over the 2x2 block using +// only the pixels covered by the layer, then blended with the same average +// alpha. This reproduces the renderer contract: the overlay sidecar is drawn +// above the zoom-blurred video layout. The launch rectangle may be the full +// layer (dynamic layers) or a one-time alpha bound (static single-frame +// layers); threads are indexed by region-local coordinates and mapped back to +// layer-local coordinates, so a bounded launch visits exactly the pixels the +// full-frame launch could write. +__global__ void blendOverlayRgbaNv12Kernel( + const unsigned char* overlay, + int overlayWidth, + int overlayHeight, + unsigned char* dst, + int dstPitch, + int dstChromaOffset, + int dstWidth, + int dstHeight, + int layerX, + int layerY, + int layerWidth, + int layerHeight, + int regionX, + int regionY, + int regionWidth, + int regionHeight) { + const int localX = blockIdx.x * blockDim.x + threadIdx.x; + const int localY = blockIdx.y * blockDim.y + threadIdx.y; + if (localX >= regionWidth || localY >= regionHeight) { + return; + } + + const int layerLocalX = regionX + localX; + const int layerLocalY = regionY + localY; + if (layerLocalX < 0 || layerLocalY < 0 || + layerLocalX >= layerWidth || layerLocalY >= layerHeight) { + return; + } + + const int x = layerX + layerLocalX; + const int y = layerY + layerLocalY; + if (x < 0 || y < 0 || x >= dstWidth || y >= dstHeight) { + return; + } + + const int pixelOffset = (layerLocalY * layerWidth + layerLocalX) * 4; + const int alpha = overlay[pixelOffset + 3]; + if (alpha > 0) { + const int r = overlay[pixelOffset]; + const int g = overlay[pixelOffset + 1]; + const int b = overlay[pixelOffset + 2]; + unsigned char overlayY = 0; + unsigned char overlayU = 0; + unsigned char overlayV = 0; + rgbaToNv12Yuv(r, g, b, overlayY, overlayU, overlayV); + unsigned char* yPtr = dst + y * dstPitch + x; + *yPtr = blendByte(*yPtr, overlayY, alpha); + } + + if ((x % 2) == 0 && (y % 2) == 0 && x + 1 < dstWidth && y + 1 < dstHeight) { + int alphaSum = 0; + int uSum = 0; + int vSum = 0; + int samples = 0; + for (int dy = 0; dy < 2; ++dy) { + for (int dx = 0; dx < 2; ++dx) { + const int sampleX = x + dx; + const int sampleY = y + dy; + const int layerLocalX = sampleX - layerX; + const int layerLocalY = sampleY - layerY; + if (layerLocalX < 0 || layerLocalY < 0 || + layerLocalX >= layerWidth || layerLocalY >= layerHeight) { + continue; + } + const int sampleOffset = (layerLocalY * layerWidth + layerLocalX) * 4; + const int sampleAlpha = overlay[sampleOffset + 3]; + if (sampleAlpha <= 0) { + continue; + } + const int r = overlay[sampleOffset]; + const int g = overlay[sampleOffset + 1]; + const int b = overlay[sampleOffset + 2]; + unsigned char sampleYValue = 0; + unsigned char sampleU = 0; + unsigned char sampleV = 0; + rgbaToNv12Yuv(r, g, b, sampleYValue, sampleU, sampleV); + alphaSum += sampleAlpha; + uSum += static_cast(sampleU) * sampleAlpha; + vSum += static_cast(sampleV) * sampleAlpha; + ++samples; + } + } + if (samples > 0) { + const int avgAlpha = alphaSum / samples; + const int avgU = uSum / alphaSum; + const int avgV = vSum / alphaSum; + unsigned char* uvPtr = dst + dstChromaOffset + (y / 2) * dstPitch + x; + uvPtr[0] = blendByte( + uvPtr[0], + static_cast(clampByteDevice(avgU)), + avgAlpha); + uvPtr[1] = blendByte( + uvPtr[1], + static_cast(clampByteDevice(avgV)), + avgAlpha); + } + } +} + +// Accumulates one weighted temporal sample into the composed frame using the +// renderer's cos-tapered shutter plan: dst = clamp(dst + (weightFixed*src)>>8). +// Weights are normalized to sum to 1, so the accumulation is a weighted average; +// NV12 chroma is accumulated per 2x2 block the same way the other blend kernels +// handle it. The target must start at zero (luma 0 / chroma 0) before the first +// sample. __global__ void prewarmKernel(unsigned int* state, unsigned int seed) { const unsigned int index = blockIdx.x * blockDim.x + threadIdx.x; unsigned int value = seed ^ (index * 747796405u + 2891336453u); @@ -1994,8 +3224,23 @@ void prewarmCuda(int durationMs) { checkCuda(cudaFree(state), "cudaFree prewarm"); } +// Map the high-level encoding mode to the current NVENC preset family. The +// legacy HP/HQ preset GUIDs cannot initialize on Blackwell-era drivers; the +// P1/P4/P6 presets must be paired with a valid tuningInfo (see the nvEncodeAPI +// note: "Presets P1-P7 are only supported with valid +// NV_ENC_INITIALIZE_PARAMS::tuningInfo"). GUID getNvencPresetGuid(const std::string& encodingMode) { - return encodingMode == "fast" ? NV_ENC_PRESET_HP_GUID : NV_ENC_PRESET_HQ_GUID; + if (encodingMode == "fast") { + return NV_ENC_PRESET_P1_GUID; + } + if (encodingMode == "quality") { + return NV_ENC_PRESET_P6_GUID; + } + return NV_ENC_PRESET_P4_GUID; +} + +NV_ENC_TUNING_INFO getNvencTuningInfo() { + return NV_ENC_TUNING_INFO_HIGH_QUALITY; } uint32_t getNvencMaxBitrate(uint32_t bitrate, const std::string& encodingMode) { @@ -2011,6 +3256,165 @@ uint32_t getNvencBufferSize(uint32_t bitrate, const std::string& encodingMode) { std::min(0xffffffffu, static_cast(bitrate) * multiplier)); } +// NVENC capability/version diagnostics captured before encoder creation. The +// compositor never claims codec or rate-control support the device does not +// list; the probe result feeds a minimal-first NV_ENC_CONFIG so optional fields +// (custom VBV, AQ) are only enabled when the hardware reports them. +struct NvencCapabilityProbe { + bool apiLoaded = false; + bool sessionOpened = false; + uint32_t driverMaxApiVersion = 0; + uint32_t sdkApiVersion = NVENCAPI_VERSION; + bool h264Supported = false; + bool hevcSupported = false; + int supportedRateControlModes = 0; + bool customVbvBufferSizeSupported = false; + bool asyncEncodeSupported = false; + bool temporalAqSupported = false; + int widthMax = 0; + int heightMax = 0; + int mbPerSecMax = 0; + std::string deviceName; + int cudaDriverVersion = 0; + int cudaComputeMajor = 0; + int cudaComputeMinor = 0; + std::string error; +}; + +// Which optional NVENC fields were actually applied after the capability probe. +// Reported so diagnostics never claim a feature (AQ, custom VBV) the hardware +// did not accept. +struct NvencConfigUsed { + bool customVbv = false; + bool aq = false; + std::string rcMode = "vbr"; +}; + +#if defined(_WIN32) +NvencCapabilityProbe probeNvencCapabilities(CUcontext context, GUID requestedCodecGuid) { + NvencCapabilityProbe probe; + probe.apiLoaded = false; + probe.sessionOpened = false; + + HMODULE module = LoadLibraryW(L"nvEncodeAPI64.dll"); + if (!module) { + probe.error = "nvEncodeAPI64.dll could not be loaded"; + return probe; + } + + typedef NVENCSTATUS(NVENCAPI* NvEncodeAPIGetMaxSupportedVersion_Type)(uint32_t*); + typedef NVENCSTATUS(NVENCAPI* NvEncodeAPICreateInstance_Type)(NV_ENCODE_API_FUNCTION_LIST*); + auto getMaxSupportedVersion = reinterpret_cast( + GetProcAddress(module, "NvEncodeAPIGetMaxSupportedVersion")); + auto createInstance = reinterpret_cast( + GetProcAddress(module, "NvEncodeAPICreateInstance")); + if (!getMaxSupportedVersion || !createInstance) { + probe.error = "NVENC API entry points not found"; + FreeLibrary(module); + return probe; + } + + NVENCSTATUS status = getMaxSupportedVersion(&probe.driverMaxApiVersion); + if (status != NV_ENC_SUCCESS) { + probe.error = "NvEncodeAPIGetMaxSupportedVersion failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + probe.apiLoaded = true; + + NV_ENCODE_API_FUNCTION_LIST functionList = {NV_ENCODE_API_FUNCTION_LIST_VER}; + status = createInstance(&functionList); + if (status != NV_ENC_SUCCESS) { + probe.error = "NvEncodeAPICreateInstance failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + + // Open a real NVENC session so the capability reads reflect the actual + // device. nvEncGetEncodeCaps requires a valid encoder handle; a null handle + // makes every caps query fail, which would silently degrade the encoder + // config to CBR without custom VBV or AQ. The session is opened against the + // same CUDA primary context the runtime allocations and the export encoder + // use and is destroyed before the real encoder session is created. + NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS openParams = {NV_ENC_OPEN_ENCODE_SESSION_EX_PARAMS_VER}; + openParams.deviceType = NV_ENC_DEVICE_TYPE_CUDA; + openParams.device = context; + openParams.apiVersion = NVENCAPI_VERSION; + void* encoder = nullptr; + status = functionList.nvEncOpenEncodeSessionEx(&openParams, &encoder); + if (status != NV_ENC_SUCCESS) { + probe.error = "nvEncOpenEncodeSessionEx failed: " + std::to_string(status); + FreeLibrary(module); + return probe; + } + probe.sessionOpened = true; + + auto queryCaps = [&](GUID codecGuid, NV_ENC_CAPS cap, int* value) -> bool { + NV_ENC_CAPS_PARAM capsParam = {NV_ENC_CAPS_PARAM_VER}; + capsParam.capsToQuery = cap; + return functionList.nvEncGetEncodeCaps(encoder, codecGuid, &capsParam, value) == + NV_ENC_SUCCESS; + }; + // Codec support is reported per codec: each query uses its own GUID so a + // device that only lists H.264 never reports HEVC support and vice versa. + int h264Value = 0; + int hevcValue = 0; + const bool h264CapsStatus = queryCaps(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES, &h264Value); + const bool hevcCapsStatus = queryCaps(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES, &hevcValue); + probe.h264Supported = h264CapsStatus; + probe.hevcSupported = hevcCapsStatus; + // Rate control / VBV / AQ / dimension caps are consumed by the encoder + // config for the requested output codec, so query them against that codec + // rather than always H.264. + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES, &probe.supportedRateControlModes); + int customVbv = 0; + int asyncEncode = 0; + int temporalAq = 0; + int widthMax = 0; + int heightMax = 0; + int mbPerSecMax = 0; + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE, &customVbv); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT, &asyncEncode); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ, &temporalAq); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_WIDTH_MAX, &widthMax); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_HEIGHT_MAX, &heightMax); + queryCaps(requestedCodecGuid, NV_ENC_CAPS_MB_PER_SEC_MAX, &mbPerSecMax); + probe.customVbvBufferSizeSupported = customVbv != 0; + probe.asyncEncodeSupported = asyncEncode != 0; + probe.temporalAqSupported = temporalAq != 0; + probe.widthMax = widthMax; + probe.heightMax = heightMax; + probe.mbPerSecMax = mbPerSecMax; + + char deviceName[256] = {}; + const CUresult deviceNameResult = cuDeviceGetName(deviceName, sizeof(deviceName), 0); + if (deviceNameResult == CUDA_SUCCESS) { + probe.deviceName = deviceName; + } + int computeMajor = 0; + int computeMinor = 0; + const CUresult computeResult = + cuDeviceComputeCapability(&computeMajor, &computeMinor, 0); + if (computeResult == CUDA_SUCCESS) { + probe.cudaComputeMajor = computeMajor; + probe.cudaComputeMinor = computeMinor; + } + cuDriverGetVersion(&probe.cudaDriverVersion); + + if (functionList.nvEncDestroyEncoder) { + functionList.nvEncDestroyEncoder(encoder); + } + FreeLibrary(module); + return probe; +} +#else +NvencCapabilityProbe probeNvencCapabilities(CUcontext, GUID) { + NvencCapabilityProbe probe; + probe.error = "NVENC probe is only implemented on Windows"; + return probe; +} +#endif + class NvencSink { public: NvencSink( @@ -2020,52 +3424,121 @@ public: int fps, uint32_t bitrate, const std::string& outputPath, - bool streamSync, Options layoutOptions, const WebcamFrameCache* webcamCache, const CursorTrack* cursorTrack, - const ZoomTrack* zoomTrack) + const ZoomTrack* zoomTrack, + OverlayFrameSource* overlaySource, + const NvencCapabilityProbe& capabilityProbe) : encoder_(context, width, height, NV_ENC_BUFFER_FORMAT_NV12), width_(width), height_(height), fps_(fps), - streamSync_(streamSync), layoutOptions_(layoutOptions), webcamCache_(webcamCache), cursorTrack_(cursorTrack), - zoomTrack_(zoomTrack) { + zoomTrack_(zoomTrack), + overlaySource_(overlaySource) { loadBackgroundFrame(); loadWebcamFrame(); loadCursorAtlas(); - if (streamSync_) { - checkCuda(cudaStreamCreateWithFlags(©Stream_, cudaStreamNonBlocking), "cudaStreamCreateWithFlags"); + temporalBlurSampleCount_ = layoutOptions_.temporalBlurSampleCount; + temporalBlurShutterFraction_ = layoutOptions_.temporalBlurShutterFraction; + temporalBlurWeightPower_ = layoutOptions_.temporalBlurWeightPower; + // The temporal sample plan depends only on the sample count, shutter + // fraction, weight curve power, and frame duration; cache it once instead + // of rebuilding the cos-tapered weights for every output frame. + if (temporalBlurSampleCount_ >= 3) { + temporalSamplePlan_ = buildTemporalSamplePlan( + temporalBlurSampleCount_, + temporalBlurShutterFraction_, + temporalBlurWeightPower_, + 1000000.0 / static_cast(fps_)); } + // Always use a non-blocking compositor stream: it keeps composite/zoom + // blur/overlay kernels ordered without implicitly serializing against the + // legacy default stream, and a single cudaStreamSynchronize before + // NVENC's synchronous input copy is the only per-frame sync needed. + checkCuda(cudaStreamCreateWithFlags(©Stream_, cudaStreamNonBlocking), "cudaStreamCreateWithFlags"); + checkCuda(cudaEventCreate(&compositeStartEvent_), "cudaEventCreate compositeStart"); + checkCuda(cudaEventCreate(&compositeEndEvent_), "cudaEventCreate compositeEnd"); + checkCuda(cudaEventCreate(&blurStartEvent_), "cudaEventCreate blurStart"); + checkCuda(cudaEventCreate(&blurEndEvent_), "cudaEventCreate blurEnd"); + checkCuda(cudaEventCreate(&overlayStartEvent_), "cudaEventCreate overlayStart"); + checkCuda(cudaEventCreate(&overlayEndEvent_), "cudaEventCreate overlayEnd"); + + // Query NVENC capability/version diagnostics before building the config. + // Optional fields (custom VBV, AQ) are only enabled when the device + // reports them, which avoids NV_ENC_ERR_INVALID_CALL (8) style failures on + // hardware/driver combinations that do not support the requested fields. + capabilityProbe_ = capabilityProbe; + const bool vbrSupported = + (capabilityProbe_.supportedRateControlModes & (1 << NV_ENC_PARAMS_RC_VBR)) != 0; + const bool customVbvSupported = capabilityProbe_.customVbvBufferSizeSupported; + const bool aqSupported = capabilityProbe_.temporalAqSupported; NV_ENC_INITIALIZE_PARAMS initializeParams = {NV_ENC_INITIALIZE_PARAMS_VER}; NV_ENC_CONFIG encodeConfig = {NV_ENC_CONFIG_VER}; initializeParams.encodeConfig = &encodeConfig; - encoder_.CreateDefaultEncoderParams( - &initializeParams, - NV_ENC_CODEC_H264_GUID, - getNvencPresetGuid(layoutOptions_.encodingMode)); - + const GUID codecGuid = layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_CODEC_HEVC_GUID + : NV_ENC_CODEC_H264_GUID; + // Build the encoder config explicitly instead of relying on + // nvEncGetEncodePresetConfig: on current SDK/driver combos the preset + // query can return an empty NV_ENC_CONFIG (rc=CONSTQP, no bitrate, + // chromaFormatIDC=0), which makes nvEncInitializeEncoder fail with + // NV_ENC_ERR_INVALID_PARAM (error 8) even for a valid NV12 export. The + // explicit minimal config below is valid on every supported NVENC device. + initializeParams.encodeGUID = codecGuid; + initializeParams.presetGUID = getNvencPresetGuid(layoutOptions_.encodingMode); + initializeParams.tuningInfo = getNvencTuningInfo(); + initializeParams.encodeWidth = static_cast(width); + initializeParams.encodeHeight = static_cast(height); + initializeParams.darWidth = static_cast(width); + initializeParams.darHeight = static_cast(height); + initializeParams.maxEncodeWidth = static_cast(width); + initializeParams.maxEncodeHeight = static_cast(height); + initializeParams.enablePTD = 1; initializeParams.frameRateNum = static_cast(fps); initializeParams.frameRateDen = 1; + // Async NVENC is the default on every supported device and is required for + // the compositor's stream-ordered pipeline; the capability probe reports it + // where available, but the sync fallback is never selected on failure. initializeParams.enableEncodeAsync = 1; - encodeConfig.profileGUID = NV_ENC_H264_PROFILE_HIGH_GUID; + encodeConfig.profileGUID = layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_HEVC_PROFILE_MAIN_GUID + : NV_ENC_H264_PROFILE_HIGH_GUID; encodeConfig.gopLength = static_cast(fps * 2); encodeConfig.frameIntervalP = 1; - encodeConfig.rcParams.rateControlMode = NV_ENC_PARAMS_RC_VBR; + // Minimal-first rate control: VBR when the device lists it, otherwise CBR. + encodeConfig.rcParams.rateControlMode = + vbrSupported ? NV_ENC_PARAMS_RC_VBR : NV_ENC_PARAMS_RC_CBR; encodeConfig.rcParams.averageBitRate = bitrate; - encodeConfig.rcParams.maxBitRate = getNvencMaxBitrate(bitrate, layoutOptions_.encodingMode); - encodeConfig.rcParams.vbvBufferSize = getNvencBufferSize(bitrate, layoutOptions_.encodingMode); - encodeConfig.rcParams.vbvInitialDelay = bitrate; - if (layoutOptions_.encodingMode != "fast") { + nvencConfigUsed_.customVbv = customVbvSupported; + if (customVbvSupported) { + encodeConfig.rcParams.maxBitRate = + getNvencMaxBitrate(bitrate, layoutOptions_.encodingMode); + encodeConfig.rcParams.vbvBufferSize = + getNvencBufferSize(bitrate, layoutOptions_.encodingMode); + encodeConfig.rcParams.vbvInitialDelay = bitrate; + } + nvencConfigUsed_.aq = aqSupported && layoutOptions_.encodingMode != "fast"; + if (nvencConfigUsed_.aq) { encodeConfig.rcParams.enableAQ = 1; - encodeConfig.rcParams.aqStrength = layoutOptions_.encodingMode == "quality" ? 10 : 8; + encodeConfig.rcParams.aqStrength = + layoutOptions_.encodingMode == "quality" ? 10 : 8; + } + if (layoutOptions_.outputCodec == OutputCodec::HEVC) { + encodeConfig.encodeCodecConfig.hevcConfig.idrPeriod = encodeConfig.gopLength; + encodeConfig.encodeCodecConfig.hevcConfig.chromaFormatIDC = 1; + } else { + encodeConfig.encodeCodecConfig.h264Config.idrPeriod = encodeConfig.gopLength; + encodeConfig.encodeCodecConfig.h264Config.chromaFormatIDC = 1; } - encodeConfig.encodeCodecConfig.h264Config.idrPeriod = encodeConfig.gopLength; encoder_.CreateEncoder(&initializeParams); + nvencConfigUsed_.rcMode = + encodeConfig.rcParams.rateControlMode == NV_ENC_PARAMS_RC_VBR ? "vbr" : "cbr"; + refreshCapabilityProbeFromEncoder(); output_.open(outputPath, std::ios::binary); if (!output_) { @@ -2141,12 +3614,71 @@ public: (std::abs(zoomSample.scale - 1.0) > 0.001 || std::abs(zoomSample.x) > 0.5 || std::abs(zoomSample.y) > 0.5); + const float safeZoomScale = std::max(0.01f, static_cast(zoomSample.scale)); + int blurRegionLeft = layoutOptions_.contentX; + int blurRegionTop = layoutOptions_.contentY; + int blurRegionRight = layoutOptions_.contentX + layoutOptions_.contentWidth; + int blurRegionBottom = layoutOptions_.contentY + layoutOptions_.contentHeight; + if (zoomChangesLayout) { + blurRegionLeft = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentX * safeZoomScale + zoomSample.x))); + blurRegionTop = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentY * safeZoomScale + zoomSample.y))); + blurRegionRight = std::min( + width_, + static_cast(std::ceil( + (layoutOptions_.contentX + layoutOptions_.contentWidth) * safeZoomScale + + zoomSample.x))); + blurRegionBottom = std::min( + height_, + static_cast(std::ceil( + (layoutOptions_.contentY + layoutOptions_.contentHeight) * safeZoomScale + + zoomSample.y))); + } + blurRegionRight = std::max(blurRegionLeft + 2, blurRegionRight); + blurRegionBottom = std::max(blurRegionTop + 2, blurRegionBottom); const bool useFastRoiComposite = canUseFastRoiComposite(zoomChangesLayout); const bool useLayeredStaticRoiComposite = !useFastRoiComposite && canUseLayeredStaticRoiComposite(zoomChangesLayout); + const bool useTemporalBlur = temporalBlurActive(); const auto compositeStart = std::chrono::steady_clock::now(); - if (useFastRoiComposite) { + checkCuda(cudaEventRecord(compositeStartEvent_, copyStream_), "cudaEventRecord compositeStart"); + if (useTemporalBlur) { + // Temporal zoom motion blur: re-composite the same decoded content at + // the renderer's symmetric shutter sample offsets (cos-tapered weights) + // with the camera transform interpolated from the zoom telemetry, then + // accumulate the weighted samples. This reproduces the configured + // high-level temporal sample plan natively instead of substituting the + // spatial blur. Webcam/cursor are applied once afterward (sharp) and + // the RGBA sidecar is blended last, so overlays stay crisp. + compositeTemporalBlurSamples( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + outputFrameTimeMs); + applySharpOverlays( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + webcamFrame, + cursorPosition, + cursorX, + cursorY, + cursorWidth, + cursorHeight, + useCursorAtlas, + cursorEntry, + block); + ++roiCompositeFrames_; + } else if (useFastRoiComposite) { copyNv12Kernel<<>>( srcFrame, srcPitch, @@ -2476,6 +4008,10 @@ public: static_cast(inputFrame->chromaOffsets[0]), width_, height_, + 0, + 0, + width_, + height_, layoutOptions_.contentX, layoutOptions_.contentY, layoutOptions_.contentWidth, @@ -2514,7 +4050,9 @@ public: zoomEnabled, static_cast(zoomSample.scale), static_cast(zoomSample.x), - static_cast(zoomSample.y)); + static_cast(zoomSample.y), + 0, + 0); checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel"); ++monolithicCompositeFrames_; } else { @@ -2532,10 +4070,89 @@ public: checkCuda(cudaGetLastError(), "copyNv12Kernel"); ++copyCompositeFrames_; } - if (streamSync_) { - checkCuda(cudaStreamSynchronize(copyStream_), "cudaStreamSynchronize copy"); - } else { - checkCuda(cudaDeviceSynchronize(), "cudaDeviceSynchronize"); + checkCuda(cudaEventRecord(compositeEndEvent_, copyStream_), "cudaEventRecord compositeEnd"); + if (!useTemporalBlur && zoomTrack_ && zoomSample.blurStrength > 0.001) { + checkCuda(cudaEventRecord(blurStartEvent_, copyStream_), "cudaEventRecord blurStart"); + applyZoomBlurFrame( + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + blurRegionLeft, + blurRegionTop, + blurRegionRight, + blurRegionBottom, + static_cast(zoomSample.blurCenterX), + static_cast(zoomSample.blurCenterY), + static_cast(zoomSample.blurStrength)); + checkCuda(cudaEventRecord(blurEndEvent_, copyStream_), "cudaEventRecord blurEnd"); + zoomBlurRecorded_ = true; + } + if (overlaySource_ && !overlaySource_->empty()) { + checkCuda(cudaEventRecord(overlayStartEvent_, copyStream_), "cudaEventRecord overlayStart"); + overlaySource_->beginFrame(outputFrameIndex, copyStream_); + for (size_t layerIndex = 0; layerIndex < overlaySource_->layerCount(); ++layerIndex) { + const auto& layer = overlaySource_->descriptor(layerIndex); + const OverlayBlendRegion overlayRegion = overlaySource_->blendRegion(layerIndex); + if (overlayRegion.width <= 0 || overlayRegion.height <= 0) { + // Fully transparent static layer: the bounded region is empty + // and the full-frame blend would write nothing either. + continue; + } + const dim3 overlayGrid( + (overlayRegion.width + block.x - 1) / block.x, + (overlayRegion.height + block.y - 1) / block.y); + blendOverlayRgbaNv12Kernel<<>>( + overlaySource_->frameDevicePtr(layerIndex, outputFrameIndex), + layer.width, + layer.height, + static_cast(inputFrame->inputPtr), + static_cast(inputFrame->pitch), + static_cast(inputFrame->chromaOffsets[0]), + width_, + height_, + layer.x, + layer.y, + layer.width, + layer.height, + overlayRegion.x, + overlayRegion.y, + overlayRegion.width, + overlayRegion.height); + checkCuda(cudaGetLastError(), "blendOverlayRgbaNv12Kernel"); + if (overlayRegion.bounded) { + ++overlayStaticRegionBlends_; + } + } + ++overlayBlendFrames_; + checkCuda(cudaEventRecord(overlayEndEvent_, copyStream_), "cudaEventRecord overlayEnd"); + overlayRecorded_ = true; + } + // Single per-frame synchronization on the compositor stream. Composite, + // zoom blur, and overlay blends are all queued on the copy stream, and + // NVENC's synchronous input copy needs them complete, so one + // cudaStreamSynchronize is both necessary and sufficient; the previous + // double sync (and any global device sync) added a full round trip per + // frame without improving correctness. + checkCuda(cudaStreamSynchronize(copyStream_), "cudaStreamSynchronize frame"); + // The overlay read-ahead prefetch is dispatched after the required frame + // sync. File reads run on the bounded background reader threads, so the + // encode thread does not block on disk I/O; ready pinned frames get + // their H2D copy enqueued here so the transfer overlaps NVENC. The + // uploads stay ordered before the next frame's beginFrame/blend because + // they are queued on the same non-blocking compositor stream, and the + // bounded ring never targets the slot the current blend read, so the + // next blend starts only after its frame is resident. + if (overlaySource_ && !overlaySource_->empty()) { + overlaySource_->prefetchNextFrame(outputFrameIndex, copyStream_); + } + accumulateStageGpuTime(compositeStartEvent_, compositeEndEvent_, compositeGpuMs_); + if (zoomBlurRecorded_) { + accumulateStageGpuTime(blurStartEvent_, blurEndEvent_, zoomBlurGpuMs_); + zoomBlurRecorded_ = false; + } + if (overlayRecorded_) { + accumulateStageGpuTime(overlayStartEvent_, overlayEndEvent_, overlayBlendGpuMs_); + overlayRecorded_ = false; } const auto compositeEnd = std::chrono::steady_clock::now(); compositeMs_ += elapsedMs(compositeStart, compositeEnd); @@ -2562,6 +4179,20 @@ public: checkCuda(cudaStreamDestroy(copyStream_), "cudaStreamDestroy"); copyStream_ = nullptr; } + cudaEvent_t stageEvents[] = { + compositeStartEvent_, + compositeEndEvent_, + blurStartEvent_, + blurEndEvent_, + overlayStartEvent_, + overlayEndEvent_, + }; + for (cudaEvent_t& event : stageEvents) { + if (event) { + checkCuda(cudaEventDestroy(event), "cudaEventDestroy stage"); + event = nullptr; + } + } if (backgroundDevice_) { checkCuda(cudaFree(backgroundDevice_), "cudaFree backgroundDevice"); backgroundDevice_ = nullptr; @@ -2574,6 +4205,19 @@ public: checkCuda(cudaFree(cursorAtlasDevice_), "cudaFree cursorAtlasDevice"); cursorAtlasDevice_ = nullptr; } + if (zoomBlurScratch_) { + checkCuda(cudaFree(zoomBlurScratch_), "cudaFree zoomBlurScratch"); + zoomBlurScratch_ = nullptr; + } + if (temporalWeightsDevice_) { + checkCuda(cudaFree(temporalWeightsDevice_), "cudaFree temporalWeightsDevice"); + temporalWeightsDevice_ = nullptr; + temporalWeightsDeviceCount_ = 0; + } + if (temporalBgCacheDevice_) { + checkCuda(cudaFree(temporalBgCacheDevice_), "cudaFree temporalBgCacheDevice"); + temporalBgCacheDevice_ = nullptr; + } } uint64_t outputBytes() const { @@ -2604,7 +4248,780 @@ public: return copyCompositeFrames_; } + int zoomBlurFrames() const { + return zoomBlurFrames_; + } + + int overlayBlendFrames() const { + return overlayBlendFrames_; + } + + int temporalBlurFrames() const { + return temporalBlurFrames_; + } + + int temporalBlurBgPrecomposedFrames() const { + return temporalBlurBgPrecomposedFrames_; + } + + int temporalBlurStationaryFrames() const { + return temporalBlurStationaryFrames_; + } + + int temporalBgCacheBuilds() const { + return temporalBgCacheBuilds_; + } + + int64_t temporalBgCacheHits() const { + return temporalBgCacheHits_; + } + + int64_t overlayStaticRegionBlends() const { + return overlayStaticRegionBlends_; + } + + int64_t overlayFileLoads() const { + return overlaySource_ ? overlaySource_->fileLoads() : 0; + } + + int64_t overlayCacheHits() const { + return overlaySource_ ? overlaySource_->cacheHits() : 0; + } + + int64_t overlayPinnedHits() const { + return overlaySource_ ? overlaySource_->pinnedHits() : 0; + } + + int64_t overlayReadWaits() const { + return overlaySource_ ? overlaySource_->readWaits() : 0; + } + + int64_t overlayPendingReadsPeak() const { + return overlaySource_ ? overlaySource_->pendingReadsPeak() : 0; + } + + int64_t temporalBlurSamplesTotal() const { + return temporalBlurSamplesTotal_; + } + + const NvencCapabilityProbe& capabilityProbe() const { + return capabilityProbe_; + } + + const NvencConfigUsed& nvencConfigUsed() const { + return nvencConfigUsed_; + } + + double compositeGpuMs() const { + return compositeGpuMs_; + } + + double zoomBlurGpuMs() const { + return zoomBlurGpuMs_; + } + + double overlayBlendGpuMs() const { + return overlayBlendGpuMs_; + } + + double overlayUploadMs() const { + return overlaySource_ ? overlaySource_->uploadMs() : 0.0; + } + + double overlayHostReadMs() const { + return overlaySource_ ? overlaySource_->hostReadMs() : 0.0; + } + + double overlayH2DEnqueueMs() const { + return overlaySource_ ? overlaySource_->h2dEnqueueMs() : 0.0; + } + private: + void refreshCapabilityProbeFromEncoder() { + // The probe session can fail caps queries on some driver/GPU combos + // (NV_ENC_ERR_ENCODER_NOT_INITIALIZED on the probe session even though + // the real encoder session works). Re-query the caps through the live + // encoder session so diagnostics report what the device truly supports + // instead of conservative fallbacks. Codec support is re-queried per + // codec (H.264 caps for h264Supported, HEVC caps for hevcSupported); + // the rate-control/VBV/AQ/dimension caps are re-queried for the codec + // the live encoder was created with. + const GUID codecGuid = codecGuidForEncoder(); + auto queryCap = [&](GUID queryCodecGuid, NV_ENC_CAPS cap) -> int { + return encoder_.GetCapabilityValue(queryCodecGuid, cap); + }; + capabilityProbe_.h264Supported = + queryCap(NV_ENC_CODEC_H264_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES) >= 0; + capabilityProbe_.hevcSupported = + queryCap(NV_ENC_CODEC_HEVC_GUID, NV_ENC_CAPS_NUM_MAX_BFRAMES) >= 0; + const int rcModes = queryCap(codecGuid, NV_ENC_CAPS_SUPPORTED_RATECONTROL_MODES); + if (rcModes >= 0) { + capabilityProbe_.supportedRateControlModes = rcModes; + } + capabilityProbe_.customVbvBufferSizeSupported = + queryCap(codecGuid, NV_ENC_CAPS_SUPPORT_CUSTOM_VBV_BUF_SIZE) > 0; + capabilityProbe_.asyncEncodeSupported = + queryCap(codecGuid, NV_ENC_CAPS_ASYNC_ENCODE_SUPPORT) > 0; + capabilityProbe_.temporalAqSupported = + queryCap(codecGuid, NV_ENC_CAPS_SUPPORT_TEMPORAL_AQ) > 0; + capabilityProbe_.widthMax = queryCap(codecGuid, NV_ENC_CAPS_WIDTH_MAX); + capabilityProbe_.heightMax = queryCap(codecGuid, NV_ENC_CAPS_HEIGHT_MAX); + capabilityProbe_.mbPerSecMax = queryCap(codecGuid, NV_ENC_CAPS_MB_PER_SEC_MAX); + } + + GUID codecGuidForEncoder() const { + return layoutOptions_.outputCodec == OutputCodec::HEVC + ? NV_ENC_CODEC_HEVC_GUID + : NV_ENC_CODEC_H264_GUID; + } + + void accumulateStageGpuTime( + cudaEvent_t startEvent, + cudaEvent_t endEvent, + double& accumulator) { + if (!startEvent || !endEvent) { + return; + } + float elapsed = 0.0f; + checkCuda( + cudaEventElapsedTime(&elapsed, startEvent, endEvent), + "cudaEventElapsedTime stage"); + accumulator += elapsed; + } + + bool temporalBlurActive() const { + return temporalBlurSampleCount_ > 0 && + zoomTrack_ != nullptr && + hasStaticLayout(layoutOptions_); + } + + void compositeTemporalBlurSamples( + unsigned char* target, + int targetPitch, + int targetChromaOffset, + const unsigned char* srcFrame, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, + double outputFrameTimeMs) { + const std::vector& samples = temporalSamplePlan_; + + const dim3 block(16, 16); + const dim3 grid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + + const bool hasSourceCrop = + layoutOptions_.sourceCropWidth >= 2 && + layoutOptions_.sourceCropHeight >= 2; + const int sourceCropX = hasSourceCrop + ? std::max(0, std::min(layoutOptions_.sourceCropX, srcWidth - 2)) & ~1 + : 0; + const int sourceCropY = hasSourceCrop + ? std::max(0, std::min(layoutOptions_.sourceCropY, srcHeight - 2)) & ~1 + : 0; + const int sourceCropWidth = hasSourceCrop + ? std::max(2, std::min(layoutOptions_.sourceCropWidth, srcWidth - sourceCropX)) & ~1 + : srcWidth; + const int sourceCropHeight = hasSourceCrop + ? std::max(2, std::min(layoutOptions_.sourceCropHeight, srcHeight - sourceCropY)) & ~1 + : srcHeight; + + // Fused composite + accumulate: the first sample replaces the target (the + // NVENC input buffer is not pre-zeroed), later samples saturate-accumulate + // the same (weight * value + 128) >> 8 math the previous two-pass + // fill/composite/accumulate pipeline produced. One pass per sample and no + // scratch buffer. The legacy path composites the full frame per sample; + // the background-precompose path below restricts per-sample work to the + // changing content region once the invariant background is accumulated. + int bboxLeft = width_; + int bboxTop = height_; + int bboxRight = 0; + int bboxBottom = 0; + bool anyContentVisible = false; + // Exact stationary shutter-window detection: when every sample resolves + // to a bit-identical camera transform (scale/x/y), every pixel selects + // the same source/layout value for every sample, so the weighted + // temporal accumulation can evaluate source + layout once per pixel and + // apply the existing fixed-point weights in order (see + // compositeStaticStationaryNv12Kernel). Any double inequality is + // enough to fall back to the per-sample paths; the comparison is exact + // so the fused path can never be chosen when per-sample transforms + // differ (even by one ULP). + bool stationaryWindow = !samples.empty(); + double stationaryScale = 0.0; + double stationaryX = 0.0; + double stationaryY = 0.0; + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + if (index == 0) { + stationaryScale = sample.scale; + stationaryX = sample.x; + stationaryY = sample.y; + } else if ( + sample.scale != stationaryScale || + sample.x != stationaryX || + sample.y != stationaryY) { + stationaryWindow = false; + } + const float safeZoomScale = std::max(0.01f, static_cast(sample.scale)); + const int transformedLeft = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentX * safeZoomScale + sample.x))); + const int transformedTop = std::max( + 0, + static_cast(std::floor(layoutOptions_.contentY * safeZoomScale + sample.y))); + const int transformedRight = std::min( + width_, + static_cast(std::ceil( + (layoutOptions_.contentX + layoutOptions_.contentWidth) * safeZoomScale + + sample.x))); + const int transformedBottom = std::min( + height_, + static_cast(std::ceil( + (layoutOptions_.contentY + layoutOptions_.contentHeight) * safeZoomScale + + sample.y))); + if (transformedRight > transformedLeft && transformedBottom > transformedTop) { + anyContentVisible = true; + bboxLeft = std::min(bboxLeft, transformedLeft); + bboxTop = std::min(bboxTop, transformedTop); + bboxRight = std::max(bboxRight, transformedRight); + bboxBottom = std::max(bboxBottom, transformedBottom); + } + } + if (stationaryWindow) { + // Every sample applies the identical transform, so sample 0's + // transform (captured by the detection loop) is the per-sample + // transform the original loop used for every sample. The fused + // kernel reproduces the replace-then-accumulate chain exactly while + // evaluating source + layout once. + ensureTemporalWeightsDevice(); + const bool sampleZoomEnabled = zoomTrack_ && stationaryScale > 0.01; + compositeStaticStationaryNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + 0, + 0, + width_, + height_, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + sampleZoomEnabled, + static_cast(stationaryScale), + static_cast(stationaryX), + static_cast(stationaryY), + temporalWeightsDevice_, + static_cast(samples.size())); + checkCuda(cudaGetLastError(), "compositeStaticStationaryNv12Kernel"); + temporalBlurSamplesTotal_ += static_cast(samples.size()); + ++temporalBlurFrames_; + ++temporalBlurStationaryFrames_; + return; + } + + // The region passed to the per-sample composite must cover the UV corner + // pixels: a UV block at even x is decided by the corner (x + 1, y + 1), + // which can map inside the content rect one pixel beyond the luma bbox + // (e.g. at an odd bbox edge). Expand the bounding box by one pixel on + // every side (clamped to the frame) so corner-driven chroma blocks get + // the same per-sample content/bg selection the full-frame kernel makes. + // Pixels inside the expansion that are background for every sample are + // recomputed identically by the region kernel, so the expansion is + // exact and only adds one boundary row/column of work. + const int regionLeft = std::max(0, bboxLeft - 1); + const int regionTop = std::max(0, bboxTop - 1); + const int regionRight = std::min(width_, bboxRight + 1); + const int regionBottom = std::min(height_, bboxBottom + 1); + const int regionWidth = regionRight - regionLeft; + const int regionHeight = regionBottom - regionTop; + const bool useBackgroundPrecompose = + !samples.empty() && + layoutOptions_.shadowIntensityPct == 0 && + (!anyContentVisible || + (regionWidth > 0 && regionHeight > 0 && + // Sample-count-aware break-even gate: the precompose path costs one + // full-frame background pass plus sampleCount region passes, while + // the per-sample path costs sampleCount full-frame passes, so + // precompose wins when 1 + N*r < N with r the region/frame area + // ratio, i.e. regionArea * N < frameArea * (N - 1). Both paths are + // bit-identical; the gate only trades GPU work. + static_cast(regionWidth) * regionHeight * + static_cast(samples.size()) < + static_cast(width_) * height_ * + static_cast(samples.size() - 1))); + if (useBackgroundPrecompose) { + compositeTemporalBlurSamplesWithBackgroundPrecompose( + target, + targetPitch, + targetChromaOffset, + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + samples, + block, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + regionLeft, + regionTop, + regionWidth, + regionHeight, + anyContentVisible, + outputFrameTimeMs); + ++temporalBlurFrames_; + ++temporalBlurBgPrecomposedFrames_; + return; + } + + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + const bool sampleZoomEnabled = zoomTrack_ && sample.scale > 0.01; + const unsigned int weightFixed = static_cast( + std::lround(samples[index].weight * 256.0)); + compositeStaticNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + 0, + 0, + width_, + height_, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + false, + false, + 0, + 0, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + sampleZoomEnabled, + static_cast(sample.scale), + static_cast(sample.x), + static_cast(sample.y), + weightFixed, + index == 0 ? 1 : 2); + checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel temporal sample"); + temporalBlurSamplesTotal_ += 1; + } + ++temporalBlurFrames_; + } + + // Fast temporal-blur path: the invariant weighted background is accumulated + // once per export into a persistent NV12 cache (see + // ensureTemporalBackgroundCache); each output frame copies that cache into + // the target and then composites only the changing content bounding box per + // temporal sample. Pixels outside the bbox are outside the content rect for + // every sample, so their per-sample value is always the background and the + // cached accumulation is exact. Inside the bbox, sample 0 replaces the + // cached background (the target is not pre-zeroed) and later samples + // saturate-accumulate, preserving the exact replace-then-accumulate + // contract of the per-sample full-frame composites term-for-term. The + // cache copy is bit-identical to the previous per-frame accumulate because + // accumulateBackgroundNv12Kernel replaces (never accumulates into) each + // destination pixel. + void compositeTemporalBlurSamplesWithBackgroundPrecompose( + unsigned char* target, + int targetPitch, + int targetChromaOffset, + const unsigned char* srcFrame, + int srcPitch, + int srcWidth, + int srcHeight, + int srcSurfaceHeight, + const std::vector& samples, + const dim3& block, + int sourceCropX, + int sourceCropY, + int sourceCropWidth, + int sourceCropHeight, + int regionLeft, + int regionTop, + int regionWidth, + int regionHeight, + bool anyContentVisible, + double outputFrameTimeMs) { + ensureTemporalWeightsDevice(); + ensureTemporalBackgroundCache(); + checkCuda( + cudaMemcpy2DAsync( + target, + static_cast(targetPitch), + temporalBgCacheDevice_, + static_cast(width_), + static_cast(width_), + static_cast(height_), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync temporal background cache Y"); + checkCuda( + cudaMemcpy2DAsync( + target + targetChromaOffset, + static_cast(targetPitch), + temporalBgCacheDevice_ + static_cast(width_) * static_cast(height_), + static_cast(width_), + static_cast(width_), + static_cast(height_ / 2), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync temporal background cache UV"); + ++temporalBgCacheHits_; + + if (!anyContentVisible || regionWidth <= 0 || regionHeight <= 0) { + return; + } + const dim3 contentGrid( + (regionWidth + block.x - 1) / block.x, + (regionHeight + block.y - 1) / block.y); + for (size_t index = 0; index < samples.size(); ++index) { + const double sampleTimeMs = outputFrameTimeMs + samples[index].offsetUs / 1000.0; + const ZoomSample sample = zoomTrack_ ? zoomTrack_->sampleAt(sampleTimeMs) : ZoomSample{}; + const bool sampleZoomEnabled = zoomTrack_ && sample.scale > 0.01; + const unsigned int weightFixed = static_cast( + std::lround(samples[index].weight * 256.0)); + compositeStaticNv12Kernel<<>>( + srcFrame, + srcPitch, + srcWidth, + srcHeight, + srcSurfaceHeight, + target, + targetPitch, + targetChromaOffset, + width_, + height_, + regionLeft, + regionTop, + regionWidth, + regionHeight, + layoutOptions_.contentX, + layoutOptions_.contentY, + layoutOptions_.contentWidth, + layoutOptions_.contentHeight, + sourceCropX, + sourceCropY, + sourceCropWidth, + sourceCropHeight, + layoutOptions_.radius, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + layoutOptions_.shadowOffsetY, + layoutOptions_.shadowIntensityPct, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + false, + false, + 0, + 0, + 0, + 0, + nullptr, + 0, + 0, + 0, + 0, + 0, + 0, + sampleZoomEnabled, + static_cast(sample.scale), + static_cast(sample.x), + static_cast(sample.y), + weightFixed, + index == 0 ? 1 : 2); + checkCuda(cudaGetLastError(), "compositeStaticNv12Kernel temporal sample region"); + temporalBlurSamplesTotal_ += 1; + } + } + + // Builds the persistent NV12 cache of the invariant weighted background + // accumulation once per export. The accumulated background depends only on + // the fixed sample weight plan and the fixed background (color or NV12 + // sidecar), so it is identical for every temporal-blur output frame; the + // first precomposed frame fills the cache with the same + // accumulateBackgroundNv12Kernel pass the previous code ran per frame, and + // later frames copy the cache into the target instead of re-accumulating. + // The cache is tightly packed (pitch == width), so the per-frame copy is + // two ordered cudaMemcpy2DAsync calls on the compositor stream. The copy is + // term-for-term identical to the per-frame accumulate because + // accumulateBackgroundNv12Kernel replaces (never accumulates into) each + // destination pixel. + void ensureTemporalBackgroundCache() { + if (temporalBgCacheDevice_) { + return; + } + const size_t requiredBytes = + static_cast(width_) * static_cast(height_) * 3 / 2; + checkCuda(cudaMalloc(&temporalBgCacheDevice_, requiredBytes), "cudaMalloc temporalBgCacheDevice"); + const dim3 block(16, 16); + const dim3 fullGrid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + accumulateBackgroundNv12Kernel<<>>( + temporalBgCacheDevice_, + width_, + static_cast(static_cast(width_) * static_cast(height_)), + width_, + height_, + clampByte(layoutOptions_.backgroundY), + clampByte(layoutOptions_.backgroundU), + clampByte(layoutOptions_.backgroundV), + backgroundDevice_, + temporalWeightsDevice_, + static_cast(temporalSamplePlan_.size())); + checkCuda(cudaGetLastError(), "accumulateBackgroundNv12Kernel cache build"); + ++temporalBgCacheBuilds_; + } + + // Uploads the cos-tapered temporal sample weights to a device buffer once; + // accumulateBackgroundNv12Kernel needs the whole plan resident for the + // invariant-background pass and the stationary fused kernel needs it for + // the in-order fixed-point accumulation. + void ensureTemporalWeightsDevice() { + const size_t count = temporalSamplePlan_.size(); + if (count == 0 || (temporalWeightsDevice_ && temporalWeightsDeviceCount_ == count)) { + return; + } + if (temporalWeightsDevice_) { + checkCuda(cudaFree(temporalWeightsDevice_), "cudaFree temporalWeightsDevice"); + temporalWeightsDevice_ = nullptr; + } + std::vector weights(count); + for (size_t index = 0; index < count; ++index) { + weights[index] = static_cast( + std::lround(temporalSamplePlan_[index].weight * 256.0)); + } + checkCuda( + cudaMalloc(&temporalWeightsDevice_, count * sizeof(unsigned int)), + "cudaMalloc temporalWeightsDevice"); + checkCuda( + cudaMemcpy( + temporalWeightsDevice_, + weights.data(), + count * sizeof(unsigned int), + cudaMemcpyHostToDevice), + "cudaMemcpy temporalWeightsDevice"); + temporalWeightsDeviceCount_ = count; + } + + void applySharpOverlays( + unsigned char* frame, + int framePitch, + int frameChromaOffset, + const unsigned char* webcamFrame, + const CursorPosition& cursorPosition, + int cursorX, + int cursorY, + int cursorWidth, + int cursorHeight, + bool useCursorAtlas, + const CursorAtlasEntry* cursorEntry, + const dim3& block) { + if (webcamFrame && layoutOptions_.webcamSize > 0) { + const int webcamRegionX = std::max(0, layoutOptions_.webcamX - 1); + const int webcamRegionY = std::max(0, layoutOptions_.webcamY - 1); + const int webcamRegionRight = + std::min(width_, layoutOptions_.webcamX + layoutOptions_.webcamSize); + const int webcamRegionBottom = + std::min(height_, layoutOptions_.webcamY + layoutOptions_.webcamSize); + const int webcamRegionWidth = webcamRegionRight - webcamRegionX; + const int webcamRegionHeight = webcamRegionBottom - webcamRegionY; + if (webcamRegionWidth > 0 && webcamRegionHeight > 0) { + const dim3 webcamGrid( + (webcamRegionWidth + block.x - 1) / block.x, + (webcamRegionHeight + block.y - 1) / block.y); + overlayWebcamNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + webcamFrame, + webcamRegionX, + webcamRegionY, + webcamRegionWidth, + webcamRegionHeight, + layoutOptions_.webcamX, + layoutOptions_.webcamY, + layoutOptions_.webcamSize, + webcamFrameWidth(), + webcamFrameHeight(), + layoutOptions_.webcamRadius, + layoutOptions_.webcamMirror); + checkCuda(cudaGetLastError(), "overlayWebcamNv12Kernel sharp"); + } + } + + if (cursorPosition.visible && cursorWidth > 0 && cursorHeight > 0) { + const int cursorPadding = useCursorAtlas ? 4 : 2; + const int regionX = std::max(0, cursorX - cursorPadding); + const int regionY = std::max(0, cursorY - cursorPadding); + const int regionRight = std::min(width_, cursorX + cursorWidth + cursorPadding); + const int regionBottom = std::min(height_, cursorY + cursorHeight + cursorPadding); + const int regionWidth = regionRight - regionX; + const int regionHeight = regionBottom - regionY; + if (regionWidth > 0 && regionHeight > 0) { + const dim3 cursorGrid( + (regionWidth + block.x - 1) / block.x, + (regionHeight + block.y - 1) / block.y); + overlayCursorNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + regionX, + regionY, + regionWidth, + regionHeight, + cursorPosition.visible, + cursorX, + cursorY, + cursorWidth, + cursorHeight, + useCursorAtlas ? cursorAtlasDevice_ : nullptr, + cursorAtlasWidth_, + cursorAtlasHeight_, + useCursorAtlas ? cursorEntry->x : 0, + useCursorAtlas ? cursorEntry->y : 0, + useCursorAtlas ? cursorEntry->width : 0, + useCursorAtlas ? cursorEntry->height : 0); + checkCuda(cudaGetLastError(), "overlayCursorNv12Kernel sharp"); + } + } + } + + void applyZoomBlurFrame( + unsigned char* frame, + int framePitch, + int frameChromaOffset, + int regionLeft, + int regionTop, + int regionRight, + int regionBottom, + float centerX, + float centerY, + float strength) { + const size_t requiredBytes = + static_cast(framePitch) * static_cast(height_) + + static_cast(framePitch) * static_cast(height_ / 2); + if (!zoomBlurScratch_ || zoomBlurScratchBytes_ < requiredBytes) { + if (zoomBlurScratch_) { + checkCuda(cudaFree(zoomBlurScratch_), "cudaFree zoomBlurScratch"); + zoomBlurScratch_ = nullptr; + zoomBlurScratchBytes_ = 0; + } + checkCuda(cudaMalloc(&zoomBlurScratch_, requiredBytes), "cudaMalloc zoomBlurScratch"); + zoomBlurScratchBytes_ = requiredBytes; + } + + const dim3 block(16, 16); + const dim3 grid((width_ + block.x - 1) / block.x, (height_ + block.y - 1) / block.y); + zoomBlurNv12Kernel<<>>( + frame, + framePitch, + frameChromaOffset, + width_, + height_, + zoomBlurScratch_, + framePitch, + frameChromaOffset, + regionLeft, + regionTop, + regionRight, + regionBottom, + centerX, + centerY, + strength); + checkCuda(cudaGetLastError(), "zoomBlurNv12Kernel"); + + checkCuda( + cudaMemcpy2DAsync( + frame, + static_cast(framePitch), + zoomBlurScratch_, + static_cast(framePitch), + static_cast(width_), + static_cast(height_), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync zoom blur Y"); + checkCuda( + cudaMemcpy2DAsync( + frame + frameChromaOffset, + static_cast(framePitch), + zoomBlurScratch_ + static_cast(framePitch) * static_cast(height_), + static_cast(framePitch), + static_cast(width_), + static_cast(height_ / 2), + cudaMemcpyDeviceToDevice, + copyStream_), + "cudaMemcpy2DAsync zoom blur UV"); + ++zoomBlurFrames_; + } + bool canUseFastRoiComposite(bool zoomChangesLayout) const { return hasStaticLayout(layoutOptions_) && layoutOptions_.contentX == 0 && @@ -2772,7 +5189,29 @@ private: int roiCompositeFrames_ = 0; int monolithicCompositeFrames_ = 0; int copyCompositeFrames_ = 0; - bool streamSync_ = false; + int zoomBlurFrames_ = 0; + int overlayBlendFrames_ = 0; + int temporalBlurFrames_ = 0; + int temporalBlurBgPrecomposedFrames_ = 0; + int temporalBlurStationaryFrames_ = 0; + int temporalBgCacheBuilds_ = 0; + int64_t temporalBgCacheHits_ = 0; + int64_t overlayStaticRegionBlends_ = 0; + int64_t temporalBlurSamplesTotal_ = 0; + bool zoomBlurRecorded_ = false; + bool overlayRecorded_ = false; + double compositeGpuMs_ = 0.0; + double zoomBlurGpuMs_ = 0.0; + double overlayBlendGpuMs_ = 0.0; + NvencCapabilityProbe capabilityProbe_; + NvencConfigUsed nvencConfigUsed_; + int temporalBlurSampleCount_ = 0; + double temporalBlurShutterFraction_ = 0.0; + double temporalBlurWeightPower_ = 1.0; + std::vector temporalSamplePlan_; + unsigned int* temporalWeightsDevice_ = nullptr; + size_t temporalWeightsDeviceCount_ = 0; + unsigned char* temporalBgCacheDevice_ = nullptr; Options layoutOptions_; unsigned char* backgroundDevice_ = nullptr; unsigned char* webcamDevice_ = nullptr; @@ -2783,7 +5222,16 @@ private: const WebcamFrameCache* webcamCache_ = nullptr; const CursorTrack* cursorTrack_ = nullptr; const ZoomTrack* zoomTrack_ = nullptr; + OverlayFrameSource* overlaySource_ = nullptr; cudaStream_t copyStream_ = nullptr; + cudaEvent_t compositeStartEvent_ = nullptr; + cudaEvent_t compositeEndEvent_ = nullptr; + cudaEvent_t blurStartEvent_ = nullptr; + cudaEvent_t blurEndEvent_ = nullptr; + cudaEvent_t overlayStartEvent_ = nullptr; + cudaEvent_t overlayEndEvent_ = nullptr; + unsigned char* zoomBlurScratch_ = nullptr; + size_t zoomBlurScratchBytes_ = 0; }; struct CallbackEncodeState { @@ -2798,6 +5246,8 @@ struct CallbackEncodeState { const WebcamFrameCache* webcamCache = nullptr; const CursorTrack* cursorTrack = nullptr; const ZoomTrack* zoomTrack = nullptr; + OverlayFrameSource* overlaySource = nullptr; + const NvencCapabilityProbe* capabilityProbe = nullptr; ProgressReportState* progress = nullptr; bool oneFramePerMappedDisplayFrame = false; int mappedFrames = 0; @@ -2815,11 +5265,31 @@ ProgressCounters collectProgressCounters( counters.encodeMs = encodeMs; if (sink) { counters.compositeMs = sink->compositeMs(); + counters.compositeGpuMs = sink->compositeGpuMs(); + counters.zoomBlurGpuMs = sink->zoomBlurGpuMs(); + counters.overlayBlendGpuMs = sink->overlayBlendGpuMs(); + counters.overlayUploadMs = sink->overlayUploadMs(); counters.nvencMs = sink->nvencMs(); counters.packetWriteMs = sink->packetWriteMs(); counters.roiCompositeFrames = sink->roiCompositeFrames(); counters.monolithicCompositeFrames = sink->monolithicCompositeFrames(); counters.copyCompositeFrames = sink->copyCompositeFrames(); + counters.zoomBlurFrames = sink->zoomBlurFrames(); + counters.overlayBlendFrames = sink->overlayBlendFrames(); + counters.temporalBlurFrames = sink->temporalBlurFrames(); + counters.temporalBlurSamplesTotal = sink->temporalBlurSamplesTotal(); + counters.temporalBlurBgPrecomposedFrames = sink->temporalBlurBgPrecomposedFrames(); + counters.temporalBlurStationaryFrames = sink->temporalBlurStationaryFrames(); + counters.temporalBgCacheBuilds = sink->temporalBgCacheBuilds(); + counters.temporalBgCacheHits = sink->temporalBgCacheHits(); + counters.overlayStaticRegionBlends = sink->overlayStaticRegionBlends(); + counters.overlayFileLoads = sink->overlayFileLoads(); + counters.overlayCacheHits = sink->overlayCacheHits(); + counters.overlayPinnedHits = sink->overlayPinnedHits(); + counters.overlayReadWaits = sink->overlayReadWaits(); + counters.overlayPendingReadsPeak = sink->overlayPendingReadsPeak(); + counters.overlayHostReadMs = sink->overlayHostReadMs(); + counters.overlayH2DEnqueueMs = sink->overlayH2DEnqueueMs(); } if (webcamCache) { counters.webcamDecodeMs = webcamCache->decodeMs; @@ -2884,6 +5354,10 @@ void encodeMappedDisplayFrame( } if (!*state->sink) { + validateOverlayBounds( + *state->options, + outputWidthForSource(*state->options, width), + outputHeightForSource(*state->options, height)); *state->sink = std::make_unique( state->context, outputWidthForSource(*state->options, width), @@ -2891,11 +5365,12 @@ void encodeMappedDisplayFrame( state->options->fps, state->bitrate, state->options->outputPath, - state->options->streamSync, *state->options, state->webcamCache, state->cursorTrack, - state->zoomTrack); + state->zoomTrack, + state->overlaySource, + state->capabilityProbe ? *state->capabilityProbe : NvencCapabilityProbe{}); } while (*state->encodedFrames < expectedOutputFrames && *state->encodedFrames < maxOutputFrames) { @@ -2953,6 +5428,16 @@ void reportEncodingProgress( intervalMs > 0.0 && intervalFrames > 0 ? static_cast(intervalFrames) * 1000.0 / intervalMs : 0.0; const double intervalEncodeMs = std::max(0.0, counters.encodeMs - state.lastCounters.encodeMs); const double intervalCompositeMs = std::max(0.0, counters.compositeMs - state.lastCounters.compositeMs); + const double intervalCompositeGpuMs = std::max(0.0, counters.compositeGpuMs - state.lastCounters.compositeGpuMs); + const double intervalZoomBlurGpuMs = std::max(0.0, counters.zoomBlurGpuMs - state.lastCounters.zoomBlurGpuMs); + const double intervalOverlayBlendGpuMs = std::max(0.0, counters.overlayBlendGpuMs - state.lastCounters.overlayBlendGpuMs); + const double intervalOverlayUploadMs = std::max(0.0, counters.overlayUploadMs - state.lastCounters.overlayUploadMs); + const double intervalOverlayHostReadMs = std::max( + 0.0, + counters.overlayHostReadMs - state.lastCounters.overlayHostReadMs); + const double intervalOverlayH2DEnqueueMs = std::max( + 0.0, + counters.overlayH2DEnqueueMs - state.lastCounters.overlayH2DEnqueueMs); const double intervalNvencMs = std::max(0.0, counters.nvencMs - state.lastCounters.nvencMs); const double intervalPacketWriteMs = std::max(0.0, counters.packetWriteMs - state.lastCounters.packetWriteMs); const double intervalWebcamDecodeMs = std::max(0.0, counters.webcamDecodeMs - state.lastCounters.webcamDecodeMs); @@ -2965,8 +5450,40 @@ void reportEncodingProgress( std::max(0, counters.monolithicCompositeFrames - state.lastCounters.monolithicCompositeFrames); const int intervalCopyCompositeFrames = std::max(0, counters.copyCompositeFrames - state.lastCounters.copyCompositeFrames); + const int intervalZoomBlurFrames = + std::max(0, counters.zoomBlurFrames - state.lastCounters.zoomBlurFrames); + const int intervalOverlayBlendFrames = + std::max(0, counters.overlayBlendFrames - state.lastCounters.overlayBlendFrames); + const int intervalTemporalBlurFrames = + std::max(0, counters.temporalBlurFrames - state.lastCounters.temporalBlurFrames); + const int64_t intervalTemporalBlurSamples = + std::max(0, counters.temporalBlurSamplesTotal - state.lastCounters.temporalBlurSamplesTotal); + const int intervalTemporalBlurBgPrecomposedFrames = std::max( + 0, + counters.temporalBlurBgPrecomposedFrames - state.lastCounters.temporalBlurBgPrecomposedFrames); + const int intervalTemporalBlurStationaryFrames = std::max( + 0, + counters.temporalBlurStationaryFrames - state.lastCounters.temporalBlurStationaryFrames); + const int intervalTemporalBgCacheBuilds = std::max( + 0, + counters.temporalBgCacheBuilds - state.lastCounters.temporalBgCacheBuilds); + const int64_t intervalTemporalBgCacheHits = std::max( + 0, + counters.temporalBgCacheHits - state.lastCounters.temporalBgCacheHits); + const int64_t intervalOverlayStaticRegionBlends = std::max( + 0, + counters.overlayStaticRegionBlends - state.lastCounters.overlayStaticRegionBlends); + const int64_t intervalOverlayFileLoads = + std::max(0, counters.overlayFileLoads - state.lastCounters.overlayFileLoads); + const int64_t intervalOverlayCacheHits = + std::max(0, counters.overlayCacheHits - state.lastCounters.overlayCacheHits); + const int64_t intervalOverlayPinnedHits = + std::max(0, counters.overlayPinnedHits - state.lastCounters.overlayPinnedHits); + const int64_t intervalOverlayReadWaits = + std::max(0, counters.overlayReadWaits - state.lastCounters.overlayReadWaits); std::cerr << std::fixed << std::setprecision(2) - << "PROGRESS {\"currentFrame\":" << encodedFrames + << "PROGRESS {\"outputCodec\":\"" << state.outputCodec + << "\",\"currentFrame\":" << encodedFrames << ",\"totalFrames\":" << totalFrames << ",\"percentage\":" << percentage << ",\"averageFps\":" << averageFps @@ -2977,6 +5494,12 @@ void reportEncodingProgress( << ",\"intervalEncodeMs\":" << intervalEncodeMs << ",\"intervalPipelineWaitMs\":" << intervalPipelineWaitMs << ",\"intervalCompositeMs\":" << intervalCompositeMs + << ",\"intervalCompositeGpuMs\":" << intervalCompositeGpuMs + << ",\"intervalZoomBlurGpuMs\":" << intervalZoomBlurGpuMs + << ",\"intervalOverlayBlendGpuMs\":" << intervalOverlayBlendGpuMs + << ",\"intervalOverlayUploadMs\":" << intervalOverlayUploadMs + << ",\"intervalOverlayHostReadMs\":" << intervalOverlayHostReadMs + << ",\"intervalOverlayH2DEnqueueMs\":" << intervalOverlayH2DEnqueueMs << ",\"intervalNvencMs\":" << intervalNvencMs << ",\"intervalPacketWriteMs\":" << intervalPacketWriteMs << ",\"intervalWebcamDecodeMs\":" << intervalWebcamDecodeMs @@ -2984,6 +5507,20 @@ void reportEncodingProgress( << ",\"intervalRoiCompositeFrames\":" << intervalRoiCompositeFrames << ",\"intervalMonolithicCompositeFrames\":" << intervalMonolithicCompositeFrames << ",\"intervalCopyCompositeFrames\":" << intervalCopyCompositeFrames + << ",\"intervalZoomBlurFrames\":" << intervalZoomBlurFrames + << ",\"intervalOverlayBlendFrames\":" << intervalOverlayBlendFrames + << ",\"intervalTemporalBlurFrames\":" << intervalTemporalBlurFrames + << ",\"intervalTemporalBlurSamples\":" << intervalTemporalBlurSamples + << ",\"intervalTemporalBlurBgPrecomposedFrames\":" << intervalTemporalBlurBgPrecomposedFrames + << ",\"intervalTemporalBlurStationaryFrames\":" << intervalTemporalBlurStationaryFrames + << ",\"intervalTemporalBgCacheBuilds\":" << intervalTemporalBgCacheBuilds + << ",\"intervalTemporalBgCacheHits\":" << intervalTemporalBgCacheHits + << ",\"intervalOverlayStaticRegionBlends\":" << intervalOverlayStaticRegionBlends + << ",\"intervalOverlayFileLoads\":" << intervalOverlayFileLoads + << ",\"intervalOverlayCacheHits\":" << intervalOverlayCacheHits + << ",\"intervalOverlayPinnedHits\":" << intervalOverlayPinnedHits + << ",\"intervalOverlayReadWaits\":" << intervalOverlayReadWaits + << ",\"overlayPendingReadsPeak\":" << counters.overlayPendingReadsPeak << "}" << std::endl; state.lastReportAt = now; state.lastReportedFrame = encodedFrames; @@ -2993,8 +5530,11 @@ void reportEncodingProgress( } // namespace int main(int argc, char** argv) { + const char* requestedOutputCodec = "h264"; + NvencCapabilityProbe capabilityProbe; try { Options options = parseOptions(argc, argv); + requestedOutputCodec = outputCodecName(options.outputCodec); options.timelineSegments = loadTimelineMap(options.timelineMapPath); if (!options.timelineSegments.empty() && !options.callbackEncode) { fail("Timeline-map CUDA export requires --callback-encode"); @@ -3005,10 +5545,30 @@ int main(int argc, char** argv) { checkCu(cuInit(0), "cuInit"); CUdevice device = 0; checkCu(cuDeviceGet(&device, 0), "cuDeviceGet"); + // Use the primary context (shared with the CUDA runtime API used for + // buffer allocation) rather than a separate cuCtxCreate context. NVENC + // capability queries and the runtime allocations must see the same + // primary context; a detached context causes caps queries to fail with + // NV_ENC_ERR_ENCODER_NOT_INITIALIZED style errors on current drivers. CUcontext context = nullptr; - checkCu(cuCtxCreate(&context, 0, device), "cuCtxCreate"); + checkCu(cuDevicePrimaryCtxRetain(&context, device), "cuDevicePrimaryCtxRetain"); checkCu(cuCtxSetCurrent(context), "cuCtxSetCurrent"); + // Run the NVENC capability probe before any runtime-API prewarm work so + // the caps query happens on a freshly current primary context. The probe + // opens a real session and queries the caps for the requested output + // codec, so the config decisions below consume real capability reads. + capabilityProbe = probeNvencCapabilities( + context, + options.outputCodec == OutputCodec::HEVC ? NV_ENC_CODEC_HEVC_GUID : NV_ENC_CODEC_H264_GUID); prewarmCuda(options.prewarmMs); + if (!capabilityProbe.apiLoaded || !capabilityProbe.sessionOpened) { + std::cerr << "{\"success\":false,\"outputCodec\":\"" + << requestedOutputCodec + << "\",\"backend\":\"nvidia-nvenc\",\"error\":\"NVENC capability probe failed: " + << capabilityProbe.error + << "\",\"noCpuFallback\":true}" << std::endl; + return 1; + } std::ifstream input(options.inputPath, std::ios::binary); if (!input) { @@ -3021,6 +5581,11 @@ int main(int argc, char** argv) { const CursorTrack* cursorTrackPtr = cursorTrack.get(); std::unique_ptr zoomTrack = loadZoomTrack(options); const ZoomTrack* zoomTrackPtr = zoomTrack.get(); + std::unique_ptr overlaySource = + options.overlayLayers.empty() + ? nullptr + : std::make_unique(options.overlayLayers); + OverlayFrameSource* overlaySourcePtr = overlaySource.get(); const std::vector sourcePts = loadFramePts(options.sourcePtsPath); const bool useSourcePts = options.inputFrames > 0 && @@ -3060,6 +5625,7 @@ int main(int argc, char** argv) { double encodeMs = 0.0; ProgressReportState progressState; progressState.startedAt = std::chrono::steady_clock::now(); + progressState.outputCodec = outputCodecName(options.outputCodec); progressState.lastReportAt = progressState.startedAt; const int progressTotalFrames = maxCallbackOutputFrames(options); reportEncodingProgress(0, progressTotalFrames, progressState, ProgressCounters{}, true); @@ -3075,6 +5641,8 @@ int main(int argc, char** argv) { webcamCachePtr, cursorTrackPtr, zoomTrackPtr, + overlaySourcePtr, + &capabilityProbe, &progressState, useDecoderFramePolicy, 0, @@ -3124,6 +5692,10 @@ int main(int argc, char** argv) { continue; } if (!sink) { + validateOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); sink = std::make_unique( context, outputWidthForSource(options, decoder->GetWidth()), @@ -3131,11 +5703,12 @@ int main(int argc, char** argv) { options.fps, bitrate, options.outputPath, - options.streamSync, options, webcamCachePtr, cursorTrackPtr, - zoomTrackPtr); + zoomTrackPtr, + overlaySourcePtr, + capabilityProbe); } const auto encodeStart = std::chrono::steady_clock::now(); sink->encodeFrame( @@ -3170,6 +5743,10 @@ int main(int argc, char** argv) { continue; } if (!sink) { + validateOverlayBounds( + options, + outputWidthForSource(options, decoder->GetWidth()), + outputHeightForSource(options, decoder->GetHeight())); sink = std::make_unique( context, outputWidthForSource(options, decoder->GetWidth()), @@ -3177,11 +5754,12 @@ int main(int argc, char** argv) { options.fps, bitrate, options.outputPath, - options.streamSync, options, webcamCachePtr, cursorTrackPtr, - zoomTrackPtr); + zoomTrackPtr, + overlaySourcePtr, + capabilityProbe); } const auto encodeStart = std::chrono::steady_clock::now(); sink->encodeFrame( @@ -3234,6 +5812,9 @@ int main(int argc, char** argv) { << "{" << "\"success\":true," << "\"mode\":\"nvdec-cuda-nvenc-annexb\"," + << "\"outputCodec\":\"" << outputCodecName(options.outputCodec) << "\"," + << "\"elementaryStreamFormat\":\"" + << outputCodecName(options.outputCodec) << "\"," << "\"selectionStage\":\"" << (options.callbackEncode ? (useDecoderFramePolicy ? "decoder-policy-mapped-callback" : "mapped-callback") @@ -3242,7 +5823,7 @@ int main(int argc, char** argv) { << "\"sourceTimestampMode\":\"" << (useSourcePts ? "pts" : "ordinal") << "\"," << "\"timelineMap\":" << (!options.timelineSegments.empty() ? "true" : "false") << "," << "\"timelineSegments\":" << options.timelineSegments.size() << "," - << "\"syncMode\":\"" << (options.streamSync ? "stream" : "device") << "\"," + << "\"syncMode\":\"stream\"," << "\"prewarmMs\":" << options.prewarmMs << "," << "\"chunkMb\":" << options.chunkMb << "," << "\"width\":" << outputWidthForSource(options, decoder->GetWidth()) << "," @@ -3279,6 +5860,41 @@ int main(int argc, char** argv) { << "\"cursorAtlas\":" << (!options.cursorAtlasRgbaPath.empty() ? "true" : "false") << "," << "\"zoomOverlay\":" << (zoomTrackPtr ? "true" : "false") << "," << "\"zoomSamples\":" << (zoomTrackPtr ? zoomTrackPtr->samples.size() : 0) << "," + << "\"zoomBlurFrames\":" << (sink ? sink->zoomBlurFrames() : 0) << "," + << "\"overlayLayers\":" << options.overlayLayers.size() << "," + << "\"overlayBlendFrames\":" << (sink ? sink->overlayBlendFrames() : 0) << "," + << "\"temporalBlurFrames\":" << (sink ? sink->temporalBlurFrames() : 0) << "," + << "\"temporalBlurSamplesTotal\":" << (sink ? sink->temporalBlurSamplesTotal() : 0) << "," + << "\"temporalBlurBgPrecomposedFrames\":" << (sink ? sink->temporalBlurBgPrecomposedFrames() : 0) << "," + << "\"temporalBlurStationaryFrames\":" << (sink ? sink->temporalBlurStationaryFrames() : 0) << "," + << "\"temporalBgCacheBuilds\":" << (sink ? sink->temporalBgCacheBuilds() : 0) << "," + << "\"temporalBgCacheHits\":" << (sink ? sink->temporalBgCacheHits() : 0) << "," + << "\"overlayStaticRegionBlends\":" << (sink ? sink->overlayStaticRegionBlends() : 0) << "," + << "\"overlayFileLoads\":" << (sink ? sink->overlayFileLoads() : 0) << "," + << "\"overlayCacheHits\":" << (sink ? sink->overlayCacheHits() : 0) << "," + << "\"overlayPinnedHits\":" << (sink ? sink->overlayPinnedHits() : 0) << "," + << "\"overlayReadWaits\":" << (sink ? sink->overlayReadWaits() : 0) << "," + << "\"overlayPendingReadsPeak\":" << (sink ? sink->overlayPendingReadsPeak() : 0) << "," + << "\"nvencDiagnostics\":{" + << "\"deviceName\":\"" << (sink ? sink->capabilityProbe().deviceName : "") << "\"," + << "\"cudaDriverVersion\":" << (sink ? sink->capabilityProbe().cudaDriverVersion : 0) << "," + << "\"cudaComputeMajor\":" << (sink ? sink->capabilityProbe().cudaComputeMajor : 0) << "," + << "\"cudaComputeMinor\":" << (sink ? sink->capabilityProbe().cudaComputeMinor : 0) << "," + << "\"sdkApiVersion\":" << (sink ? sink->capabilityProbe().sdkApiVersion : 0) << "," + << "\"driverMaxApiVersion\":" << (sink ? sink->capabilityProbe().driverMaxApiVersion : 0) << "," + << "\"h264Supported\":" << (sink && sink->capabilityProbe().h264Supported ? "true" : "false") << "," + << "\"hevcSupported\":" << (sink && sink->capabilityProbe().hevcSupported ? "true" : "false") << "," + << "\"supportedRateControlModes\":" << (sink ? sink->capabilityProbe().supportedRateControlModes : 0) << "," + << "\"customVbvSupported\":" << (sink && sink->capabilityProbe().customVbvBufferSizeSupported ? "true" : "false") << "," + << "\"asyncEncodeSupported\":" << (sink && sink->capabilityProbe().asyncEncodeSupported ? "true" : "false") << "," + << "\"temporalAqSupported\":" << (sink && sink->capabilityProbe().temporalAqSupported ? "true" : "false") << "," + << "\"widthMax\":" << (sink ? sink->capabilityProbe().widthMax : 0) << "," + << "\"heightMax\":" << (sink ? sink->capabilityProbe().heightMax : 0) << "," + << "\"mbPerSecMax\":" << (sink ? sink->capabilityProbe().mbPerSecMax : 0) << "," + << "\"probeError\":\"" << (sink ? sink->capabilityProbe().error : "") << "\"," + << "\"rcModeUsed\":\"" << (sink ? sink->nvencConfigUsed().rcMode : "") << "\"," + << "\"customVbvUsed\":" << (sink && sink->nvencConfigUsed().customVbv ? "true" : "false") << "," + << "\"aqUsed\":" << (sink && sink->nvencConfigUsed().aq ? "true" : "false") << "}," << "\"sourceFrames\":" << reportedSourceFrames << "," << "\"mappedDisplayFrames\":" << mappedDisplayFrames << "," << "\"selectedDisplayFrames\":" << selectedDisplayFrames << "," @@ -3289,6 +5905,12 @@ int main(int argc, char** argv) { << "\"decodeWallMs\":" << decodeMs << "," << "\"encodeMs\":" << encodeMs << "," << "\"compositeMs\":" << sink->compositeMs() << "," + << "\"compositeGpuMs\":" << sink->compositeGpuMs() << "," + << "\"zoomBlurGpuMs\":" << sink->zoomBlurGpuMs() << "," + << "\"overlayBlendGpuMs\":" << sink->overlayBlendGpuMs() << "," + << "\"overlayUploadMs\":" << sink->overlayUploadMs() << "," + << "\"overlayHostReadMs\":" << sink->overlayHostReadMs() << "," + << "\"overlayH2DEnqueueMs\":" << sink->overlayH2DEnqueueMs() << "," << "\"roiCompositeFrames\":" << sink->roiCompositeFrames() << "," << "\"monolithicCompositeFrames\":" << sink->monolithicCompositeFrames() << "," << "\"copyCompositeFrames\":" << sink->copyCompositeFrames() << "," @@ -3304,10 +5926,33 @@ int main(int argc, char** argv) { sink.reset(); decoder.reset(); webcamStream.reset(); - checkCu(cuCtxDestroy(context), "cuCtxDestroy"); + // OverlayFrameSource owns device/pinned buffers (cudaFree/cudaFreeHost in + // its destructor); it must be destroyed while the primary context is still + // current, before the context is released. + overlaySource.reset(); + // The primary context is released, not destroyed. + checkCu(cuDevicePrimaryCtxRelease(device), "cuDevicePrimaryCtxRelease"); return 0; } catch (const std::exception& error) { - std::cerr << "{\"success\":false,\"error\":\"" << error.what() << "\"}" << std::endl; + std::cerr << "{\"success\":false,\"outputCodec\":\"" + << requestedOutputCodec + << "\",\"backend\":\"nvidia-nvenc\",\"error\":\"" + << error.what() + << "\",\"nvencDiagnostics\":{" + << "\"deviceName\":\"" << capabilityProbe.deviceName << "\"," + << "\"cudaDriverVersion\":" << capabilityProbe.cudaDriverVersion << "," + << "\"cudaComputeMajor\":" << capabilityProbe.cudaComputeMajor << "," + << "\"cudaComputeMinor\":" << capabilityProbe.cudaComputeMinor << "," + << "\"sdkApiVersion\":" << capabilityProbe.sdkApiVersion << "," + << "\"driverMaxApiVersion\":" << capabilityProbe.driverMaxApiVersion << "," + << "\"h264Supported\":" << (capabilityProbe.h264Supported ? "true" : "false") << "," + << "\"hevcSupported\":" << (capabilityProbe.hevcSupported ? "true" : "false") << "," + << "\"supportedRateControlModes\":" << capabilityProbe.supportedRateControlModes << "," + << "\"customVbvSupported\":" << (capabilityProbe.customVbvBufferSizeSupported ? "true" : "false") << "," + << "\"asyncEncodeSupported\":" << (capabilityProbe.asyncEncodeSupported ? "true" : "false") << "," + << "\"temporalAqSupported\":" << (capabilityProbe.temporalAqSupported ? "true" : "false") << "," + << "\"probeError\":\"" << capabilityProbe.error << "\"}," + << "\"noCpuFallback\":true}" << std::endl; return 1; } } diff --git a/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs b/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs new file mode 100644 index 000000000..41c02cde4 --- /dev/null +++ b/electron/native/nvidia-cuda-compositor/temporalAccumulate.test.mjs @@ -0,0 +1,529 @@ +import { describe, expect, it } from "vitest"; + +// Contract test for the fused temporal composite + accumulate in main.cu +// (compositeStaticNv12Kernel temporalWeightFixed/temporalAccumulateMode). The +// native kernel replaced the two-pass fill/composite/accumulate pipeline with a +// single fused pass: sample 0 replaces the target with (w0 * v + 128) >> 8 and +// later samples saturate-accumulate (w * v + 128) >> 8. This JS mirror verifies +// the fused order reproduces the previous zero-fill + saturating adds exactly, +// including the fixed-point weight plan used by buildTemporalSamplePlan. +// +// Module 3 contracts (constants synchronized with current CUDA math in +// electron/native/nvidia-cuda-compositor/src/main.cu): +// - stationary-transform fixed-point accumulation equivalence: with an +// invariant per-sample transform, the fused full-frame path, the legacy +// zero-fill path, and the background-precompose path (accumulateBackgroundNv12Kernel +// one-pass term chain + region re-composite) are bit-identical. +// - sample-count-aware precompose threshold: the cost-model break-even region +// ratio is (sampleCount - 1) / sampleCount, while the CUDA decision keeps a +// fixed 19/20 region gate (regionPixels * 20 < framePixels * 19) plus the +// empty-sample and shadowIntensityPct == 0 gates. +// - a stationary-window predicate mirror for the invariant-transform property +// the compositor exploits implicitly (per-sample content rects collapse to +// one rect, so the UV-expanded region is identical every sample). + +function buildTemporalSamplePlan(sampleCount, shutterFraction, weightCurvePower, frameDurationUs) { + const safeSampleCount = Math.max(1, sampleCount); + if (safeSampleCount <= 1) { + return [{ offsetUs: 0.0, weight: 1.0 }]; + } + const shutterWindowUs = + Math.max(1.0, frameDurationUs) * Math.max(0.0, Math.min(3.0, shutterFraction)); + const startOffsetUs = -shutterWindowUs / 2.0; + const stepUs = shutterWindowUs / (safeSampleCount - 1); + const offsetsUs = []; + for (let index = 0; index < safeSampleCount; index += 1) { + offsetsUs.push(startOffsetUs + stepUs * index); + } + const kWeightFloor = 0.22; + const centerIndex = (safeSampleCount - 1) / 2; + const rawWeights = []; + let totalWeight = 0; + for (let index = 0; index < safeSampleCount; index += 1) { + const normalizedDistance = Math.abs(index - centerIndex) / Math.max(1, centerIndex); + const taperedWeight = Math.cos(normalizedDistance * (Math.PI / 2)); + const rawWeight = + kWeightFloor + + (1 - kWeightFloor) * Math.pow(Math.max(0, taperedWeight), weightCurvePower); + rawWeights.push(rawWeight); + totalWeight += rawWeight; + } + return offsetsUs.map((offsetUs, index) => ({ + offsetUs, + weight: totalWeight > 0 ? rawWeights[index] / totalWeight : 1 / safeSampleCount, + })); +} + +function temporalAccumulateByte(current, value, weightFixed, mode) { + if (mode === 0) { + return value; + } + const weighted = (weightFixed * value + 128) >> 8; + if (mode === 1) { + return Math.min(255, weighted); + } + return Math.min(255, current + weighted); +} + +// Old two-pass: zero-fill target, then per sample add (w * scratch + 128) >> 8. +function oldAccumulate(values, weights) { + let target = 0; + for (let index = 0; index < values.length; index += 1) { + const weightFixed = Math.round(weights[index] * 256); + target = Math.min(255, target + ((weightFixed * values[index] + 128) >> 8)); + } + return target; +} + +// New fused: sample 0 replaces with (w0 * v + 128) >> 8, rest saturate-accumulate. +function fusedAccumulate(values, weights) { + let target = 0; + for (let index = 0; index < values.length; index += 1) { + const weightFixed = Math.round(weights[index] * 256); + target = temporalAccumulateByte(target, values[index], weightFixed, index === 0 ? 1 : 2); + } + return target; +} + +// Mirrors ensureTemporalWeightsDevice in main.cu: weights are rounded to 8-bit +// fixed point once (std::lround(weight * 256.0)) and reused by every +// accumulate pass, so per-sample rounding is identical across paths. +function fixedWeights(weights) { + return weights.map((weight) => Math.round(weight * 256)); +} + +// Mirrors accumulateBackgroundNv12Kernel in main.cu: sample 0 seeds the +// accumulator with (w0 * v + 128) >> 8 and every later sample saturate-adds +// (w * v + 128) >> 8 (min(255, acc + term)). This is the kernel's one-pass +// invariant-background accumulation; for a stationary transform it is exactly +// what the fused and legacy full-frame paths produce per pixel. +function saturatingAccumulate(value, weights) { + if (weights.length === 0) { + return 0; + } + const fixed = fixedWeights(weights); + let acc = (fixed[0] * value + 128) >> 8; + for (let index = 1; index < fixed.length; index += 1) { + acc = Math.min(255, acc + ((fixed[index] * value + 128) >> 8)); + } + return acc; +} + +// Mirrors the useBackgroundPrecompose decision in compositeTemporalBlurSamples: +// precompose requires at least one sample, no shadow compositing +// (shadowIntensityPct == 0), and either no visible content or a positive +// UV-expanded region strictly smaller than 19/20 of the frame +// (regionPixels * 20 < framePixels * 19). The 20/19 constants are the current +// CUDA math; the sample-count-aware cost model is mirrored separately below. +function shouldUseBackgroundPrecompose({ + sampleCount, + shadowIntensityPct, + anyContentVisible, + regionWidth, + regionHeight, + frameWidth, + frameHeight, +}) { + return ( + sampleCount > 0 && + shadowIntensityPct === 0 && + (!anyContentVisible || + (regionWidth > 0 && + regionHeight > 0 && + regionWidth * regionHeight * 20 < frameWidth * frameHeight * 19)) + ); +} + +// Cost model behind the precompose choice: the legacy path composites the full +// frame per sample (sampleCount * framePixels); precompose runs one full-frame +// background pass plus one region pass per sample +// (framePixels + sampleCount * regionPixels). Precompose wins strictly when +// regionPixels < framePixels * (sampleCount - 1) / sampleCount. The integer +// form keeps the strict-inequality boundary exact for every supported sample +// count (the CUDA option gate accepts 3..61 samples). +function precomposeWinsCostModel(sampleCount, regionPixels, framePixels) { + return regionPixels * sampleCount < framePixels * (sampleCount - 1); +} + +function precomposeBreakEvenRegionRatio(sampleCount) { + return (sampleCount - 1) / sampleCount; +} + +// Stationary-window predicate mirror. The CUDA compositor has no dedicated +// stationary predicate: invariance is implicit because every sample composites +// the same transform, so the per-sample content bounding boxes collapse to one +// rect and the UV-expanded region is identical for every sample. This mirror +// models that property (scale/x/y unchanged within epsilon) for the +// stationary-equivalence tests. blurStrength/blurCenter are intentionally +// ignored: the temporal path replaces spatial blur for those frames. +function isStationarySampleWindow(samples) { + if (samples.length === 0) { + return false; + } + const first = samples[0]; + if ( + !first || + typeof first.scale !== "number" || + typeof first.x !== "number" || + typeof first.y !== "number" + ) { + return false; + } + const epsilon = 1e-9; + return samples.every( + (sample) => + Math.abs(sample.scale - first.scale) <= epsilon && + Math.abs(sample.x - first.x) <= epsilon && + Math.abs(sample.y - first.y) <= epsilon, + ); +} + +// Models one NV12 plane (luma or chroma) for a stationary transform: every +// sample composites the same invariant per-pixel value (content inside the +// region, background outside), so the whole-frame fused path, the legacy +// zero-fill path, and the background-precompose path must agree bit-for-bit. +// Returns the three results as arrays indexed by y * width + x. +function stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues, + backgroundValue, + weights, +}) { + const fullFrameFused = []; + const fullFrameOld = []; + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const invariantValue = + x < regionWidth && y < regionHeight + ? contentValues[y * width + x] + : backgroundValue; + fullFrameFused.push( + fusedAccumulate(new Array(weights.length).fill(invariantValue), weights), + ); + fullFrameOld.push( + oldAccumulate(new Array(weights.length).fill(invariantValue), weights), + ); + } + } + + // Background precompose: accumulate the invariant background once per pixel + // over the full frame (the kernel mirror), then re-composite only the region + // per sample: sample 0 replaces the precomposed background (the target is + // not pre-zeroed) and later samples saturate-accumulate. + const backgroundAcc = saturatingAccumulate(backgroundValue, weights); + const backgroundPrecompose = new Array(width * height).fill(backgroundAcc); + for (let y = 0; y < regionHeight; y += 1) { + for (let x = 0; x < regionWidth; x += 1) { + const index = y * width + x; + backgroundPrecompose[index] = fusedAccumulate( + new Array(weights.length).fill(contentValues[index]), + weights, + ); + } + } + + return { fullFrameFused, fullFrameOld, backgroundPrecompose }; +} + +function planWeights(sampleCount, shutterFraction = 0.5, weightCurvePower = 2) { + return buildTemporalSamplePlan( + sampleCount, + shutterFraction, + weightCurvePower, + 1000000 / 30, + ).map((sample) => sample.weight); +} + +describe("fused temporal accumulate contract", () => { + it("fused replace-then-accumulate equals zero-fill + saturating adds", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + const fixedSum = weights.reduce((sum, w) => sum + Math.round(w * 256), 0); + expect(fixedSum).toBeGreaterThan(250); + + // Deterministic pseudo-random values covering the full byte range. + let seed = 12345; + const values = []; + for (let i = 0; i < 13; i += 1) { + seed = (seed * 1664525 + 1013904223) >>> 0; + values.push((seed >> 16) & 0xff); + } + expect(fusedAccumulate(values, weights)).toBe(oldAccumulate(values, weights)); + }); + + it("constant luma/background values accumulate to the renderer result", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + // Constant 128 (neutral chroma) must land on the documented 131 for this plan. + expect(fusedAccumulate(new Array(13).fill(128), weights)).toBe(131); + }); + + it("keeps saturating semantics per step (no wraparound near 255)", () => { + const plan = buildTemporalSamplePlan(13, 0.5, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + const result = fusedAccumulate(new Array(13).fill(255), weights); + // Saturating adds must never wrap below 250 for an all-255 frame. + expect(result).toBeGreaterThan(250); + expect(result).toBe(oldAccumulate(new Array(13).fill(255), weights)); + }); + + it("matches the old pipeline for a range of shutter fractions and sample counts", () => { + for (const sampleCount of [3, 5, 9, 13, 17]) { + for (const shutter of [0.25, 0.5, 1.0]) { + const plan = buildTemporalSamplePlan(sampleCount, shutter, 2, 1000000 / 30); + const weights = plan.map((sample) => sample.weight); + for (const base of [0, 1, 16, 64, 128, 200, 254, 255]) { + const values = new Array(sampleCount).fill(base); + expect(fusedAccumulate(values, weights)).toBe(oldAccumulate(values, weights)); + } + } + } + }); +}); + +describe("stationary-transform fixed-point accumulation equivalence", () => { + it("fused full-frame, legacy zero-fill, and background precompose agree for a stationary window", () => { + const width = 64; + const height = 36; + const regionWidth = 40; + const regionHeight = 24; + let seed = 987654321; + const contentValues = []; + for (let index = 0; index < regionWidth * regionHeight; index += 1) { + seed = (seed * 1664525 + 1013904223) >>> 0; + contentValues.push((seed >> 16) & 0xff); + } + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + const { fullFrameFused, fullFrameOld, backgroundPrecompose } = + stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues, + backgroundValue: 16, + weights, + }); + expect(fullFrameFused).toEqual(fullFrameOld); + expect(backgroundPrecompose).toEqual(fullFrameFused); + } + }); + + it("the background accumulate kernel mirror equals the fused and legacy paths for any constant value", () => { + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + for (const backgroundValue of [0, 1, 16, 64, 128, 200, 254, 255]) { + expect(saturatingAccumulate(backgroundValue, weights)).toBe( + fusedAccumulate(new Array(sampleCount).fill(backgroundValue), weights), + ); + expect(saturatingAccumulate(backgroundValue, weights)).toBe( + oldAccumulate(new Array(sampleCount).fill(backgroundValue), weights), + ); + } + } + }); + + it("matches on chroma planes and at saturation boundaries across sample counts", () => { + const width = 32; + const height = 18; + const regionWidth = 20; + const regionHeight = 12; + for (const sampleCount of [3, 5, 13, 61]) { + const weights = planWeights(sampleCount); + const chroma = stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues: new Array(regionWidth * regionHeight).fill(128), + backgroundValue: 128, + weights, + }); + const luma = stationaryPlaneEquivalence({ + width, + height, + regionWidth, + regionHeight, + contentValues: new Array(regionWidth * regionHeight).fill(255), + backgroundValue: 16, + weights, + }); + expect(chroma.backgroundPrecompose).toEqual(chroma.fullFrameFused); + expect(luma.backgroundPrecompose).toEqual(luma.fullFrameOld); + // Saturating adds must never wrap; neutral chroma stays <= 255. + expect(Math.max(...chroma.fullFrameFused)).toBeLessThanOrEqual(255); + expect(Math.max(...luma.backgroundPrecompose)).toBeLessThanOrEqual(255); + } + }); +}); + +describe("sample-count-aware precompose threshold", () => { + it("break-even region ratio is (sampleCount - 1) / sampleCount", () => { + expect(precomposeBreakEvenRegionRatio(3)).toBe(2 / 3); + expect(precomposeBreakEvenRegionRatio(5)).toBe(4 / 5); + expect(precomposeBreakEvenRegionRatio(13)).toBe(12 / 13); + expect(precomposeBreakEvenRegionRatio(61)).toBe(60 / 61); + }); + + it("strictly favors precompose only below the break-even region for 3, 5, 13, and 61 samples", () => { + const framePixels = 1920 * 1080; + for (const sampleCount of [3, 5, 13, 61]) { + // Largest integer region strictly below the real break-even + // B = framePixels * (sampleCount - 1) / sampleCount is ceil(B) - 1; + // floor(B) is one too large when B is not an integer. + const largestBelow = Math.ceil((framePixels * (sampleCount - 1)) / sampleCount) - 1; + expect(precomposeWinsCostModel(sampleCount, largestBelow, framePixels)).toBe(true); + expect(precomposeWinsCostModel(sampleCount, largestBelow + 1, framePixels)).toBe(false); + expect(precomposeWinsCostModel(sampleCount, 0, framePixels)).toBe(true); + expect(precomposeWinsCostModel(sampleCount, framePixels, framePixels)).toBe(false); + } + }); + + it("mirrors the current CUDA 20/19 fixed region threshold", () => { + const base = { + sampleCount: 13, + shadowIntensityPct: 0, + anyContentVisible: true, + frameWidth: 1000, + frameHeight: 1000, + }; + // 949000 < 950000 (19/20 of 1000x1000): precompose. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 949, regionHeight: 1000 }), + ).toBe(true); + // Exactly 19/20 is not strictly smaller: legacy full-frame path. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 950, regionHeight: 1000 }), + ).toBe(false); + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 951, regionHeight: 1000 }), + ).toBe(false); + }); + + it("keeps the empty-sample and shadow gates from the CUDA decision", () => { + const base = { + sampleCount: 13, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 100, + regionHeight: 100, + frameWidth: 1000, + frameHeight: 1000, + }; + expect(shouldUseBackgroundPrecompose({ ...base, sampleCount: 0 })).toBe(false); + expect(shouldUseBackgroundPrecompose({ ...base, shadowIntensityPct: 40 })).toBe(false); + expect(shouldUseBackgroundPrecompose(base)).toBe(true); + // No visible content: precompose regardless of the region size. + expect( + shouldUseBackgroundPrecompose({ + ...base, + anyContentVisible: false, + regionWidth: 0, + regionHeight: 0, + }), + ).toBe(true); + // Degenerate region with visible content: legacy. + expect( + shouldUseBackgroundPrecompose({ + ...base, + anyContentVisible: true, + regionWidth: 0, + regionHeight: 0, + }), + ).toBe(false); + }); + + it("applies the fixed threshold at every supported sample count", () => { + for (const sampleCount of [3, 5, 13, 61]) { + const base = { + sampleCount, + shadowIntensityPct: 0, + anyContentVisible: true, + frameWidth: 1920, + frameHeight: 1080, + }; + // 19/20 of 1920x1080 is exactly 1920x1026. + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 1920, regionHeight: 1026 }), + ).toBe(false); + expect( + shouldUseBackgroundPrecompose({ ...base, regionWidth: 1920, regionHeight: 1025 }), + ).toBe(true); + } + }); + + it("pins the band where the fixed threshold and the cost model diverge by sample count", () => { + const framePixels = 1920 * 1080; + // 3 samples: break-even is 2/3. A 90%-of-frame region is cheaper via the + // legacy path, but the current CUDA threshold still selects precompose + // (exact output, just more region work than legacy). + expect(precomposeWinsCostModel(3, 1920 * 972, framePixels)).toBe(false); + expect( + shouldUseBackgroundPrecompose({ + sampleCount: 3, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 1920, + regionHeight: 972, + frameWidth: 1920, + frameHeight: 1080, + }), + ).toBe(true); + // 61 samples: break-even is 60/61 (~98.4%). A 96%-of-frame region is + // cheaper via precompose, but the fixed threshold keeps the legacy path. + expect(precomposeWinsCostModel(61, 1920 * 1037, framePixels)).toBe(true); + expect( + shouldUseBackgroundPrecompose({ + sampleCount: 61, + shadowIntensityPct: 0, + anyContentVisible: true, + regionWidth: 1920, + regionHeight: 1037, + frameWidth: 1920, + frameHeight: 1080, + }), + ).toBe(false); + }); +}); + +describe("stationary-window predicate mirror", () => { + it("accepts a window where every sample shares the same transform", () => { + const samples = [ + { offsetUs: -16666.666666666668, weight: 0.02918, scale: 1.25, x: 40, y: 20 }, + { offsetUs: 0, weight: 0.081, scale: 1.25, x: 40, y: 20 }, + { offsetUs: 16666.666666666668, weight: 0.02918, scale: 1.25, x: 40, y: 20 }, + ]; + expect(isStationarySampleWindow(samples)).toBe(true); + }); + + it("rejects windows with zoom or pan motion between samples", () => { + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.1, x: 0, y: 0 }, + ]), + ).toBe(false); + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.0, x: 3, y: 0 }, + ]), + ).toBe(false); + expect( + isStationarySampleWindow([ + { scale: 1.0, x: 0, y: 0 }, + { scale: 1.0, x: 0, y: -2 }, + ]), + ).toBe(false); + }); + + it("accepts a single-sample plan and rejects empty or malformed windows", () => { + expect(isStationarySampleWindow([])).toBe(false); + expect(isStationarySampleWindow([{ scale: 1.0, x: 0, y: 0 }])).toBe(true); + expect(isStationarySampleWindow([{ weight: 1.0 }])).toBe(false); + }); +}); diff --git a/electron/preload.ts b/electron/preload.ts index e55d42cbd..7fcd82988 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -1,7 +1,53 @@ import { contextBridge, ipcRenderer } from "electron"; import type { RecordingSessionData } from "./ipc/types"; -type NativeVideoExportWriteResult = { success: boolean; error?: string }; +type NativeVideoExportWriteResult = { + success: boolean; + error?: string; + fallbackAvailable?: boolean; +}; +type NativeVideoExportFrameChannelResult = NativeVideoExportWriteResult; +type NativeVideoExportFramePortResponse = + | { + type: "ready"; + protocol: 1; + sessionId: string; + transferable: true; + transferProbe: ArrayBuffer; + } + | { + type: "ack"; + protocol: 1; + sessionId: string; + requestId: number; + sequence: number; + success: true; + } + | { + type: "error"; + protocol: 1; + sessionId: string; + requestId?: number; + sequence?: number; + success: false; + error: string; + fallbackAvailable: boolean; + }; +type NativeVideoExportFrameChannelState = { + sessionId: string; + port: MessagePort; + nextSequence: number; + pending: Map< + number, + { sequence: number; resolve: (result: NativeVideoExportWriteResult) => void } + >; + ready: Promise; + resolveReady: () => void; + rejectReady: (error: Error) => void; + readySettled: boolean; + closed: boolean; + handshakeTimeout: ReturnType; +}; type NativeVideoAudioMuxMetrics = { tempVideoWriteMs?: number; tempEditedAudioWriteMs?: number; @@ -56,6 +102,10 @@ type NativeStaticLayoutChunkMetric = { outputBytes: number; fallbackReason?: string; windowsGpuSummary?: WindowsGpuExportSummary; + nvidiaCudaSummary?: { + success?: boolean; + outputCodec?: "h264" | "hevc"; + }; }; type NativeStaticLayoutMetrics = NativeVideoAudioMuxMetrics & { chunkCount: number; @@ -73,6 +123,8 @@ type NativeStaticLayoutProgress = { stage?: "preparing" | "finalizing"; elapsedMs?: number; averageFps?: number; + estimatedFps?: number; + fpsSource?: "native" | "estimated"; currentFrame: number; totalFrames: number; percentage: number; @@ -113,6 +165,7 @@ const nativeVideoExportWriteRequests = new Map< let nextNativeVideoExportWriteRequestId = 1; let nativeVideoExportWriteResultListenerAttached = false; +const nativeVideoExportFrameChannels = new Map(); function ensureNativeVideoExportWriteResultListener() { if (nativeVideoExportWriteResultListenerAttached) { @@ -163,6 +216,309 @@ function settleNativeVideoExportPendingRequests( } } +function writeNativeVideoExportFramesLegacy( + sessionId: string, + frameDataList: Uint8Array[], +): Promise { + ensureNativeVideoExportWriteResultListener(); + + return new Promise((resolve) => { + const requestId = nextNativeVideoExportWriteRequestId++; + nativeVideoExportWriteRequests.set(requestId, { + sessionId, + resolve, + }); + + ipcRenderer.send("native-video-export-write-frames-async", { + sessionId, + requestId, + frameDataList, + }); + }); +} + +function isArrayBuffer(value: unknown): value is ArrayBuffer { + return value instanceof ArrayBuffer; +} + +function isNativeVideoExportFramePortResponse( + value: unknown, +): value is NativeVideoExportFramePortResponse { + if (!value || typeof value !== "object") { + return false; + } + + const payload = value as Record; + return payload.protocol === 1 && typeof payload.type === "string"; +} + +function settleNativeVideoExportFrameChannelState( + state: NativeVideoExportFrameChannelState, + error: string, +) { + if (state.closed) { + return; + } + + state.closed = true; + clearTimeout(state.handshakeTimeout); + if (!state.readySettled) { + state.readySettled = true; + state.rejectReady(new Error(error)); + } + for (const pendingRequest of state.pending.values()) { + pendingRequest.resolve({ success: false, error }); + } + state.pending.clear(); + if (nativeVideoExportFrameChannels.get(state.sessionId) === state) { + nativeVideoExportFrameChannels.delete(state.sessionId); + } + try { + state.port.close(); + } catch { + // The main process may already have closed the port. + } +} + +function handleNativeVideoExportFramePortResponse( + state: NativeVideoExportFrameChannelState, + value: unknown, +) { + if (!isNativeVideoExportFramePortResponse(value) || value.sessionId !== state.sessionId) { + return; + } + + if (value.type === "ready") { + if ( + value.transferable !== true || + !isArrayBuffer(value.transferProbe) || + value.transferProbe.byteLength !== 1 || + state.readySettled + ) { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel returned an invalid handshake", + ); + return; + } + state.readySettled = true; + state.resolveReady(); + return; + } + + if (value.type === "ack") { + const pendingRequest = state.pending.get(value.requestId); + if (!pendingRequest) { + return; + } + state.pending.delete(value.requestId); + pendingRequest.resolve( + value.sequence === pendingRequest.sequence + ? { success: true } + : { + success: false, + error: "Native export frame acknowledgement sequence mismatch", + }, + ); + return; + } + + if (value.type === "error") { + if (typeof value.requestId === "number") { + const pendingRequest = state.pending.get(value.requestId); + if (!pendingRequest) { + return; + } + state.pending.delete(value.requestId); + pendingRequest.resolve({ + success: false, + error: value.error, + fallbackAvailable: value.fallbackAvailable, + }); + return; + } + settleNativeVideoExportFrameChannelState(state, value.error); + } +} + +function openNativeVideoExportFrameChannel( + sessionId: string, +): Promise { + const existing = nativeVideoExportFrameChannels.get(sessionId); + if (existing) { + return existing.ready + .then(() => ({ success: true })) + .catch((error: unknown) => ({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + })); + } + + if (typeof MessageChannel === "undefined" || typeof ipcRenderer.postMessage !== "function") { + return Promise.resolve({ + success: false, + error: "Native export transferable frame channels are unavailable", + fallbackAvailable: true, + }); + } + + let channel: MessageChannel; + try { + channel = new MessageChannel(); + } catch (error) { + return Promise.resolve({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + }); + } + + let resolveReady!: () => void; + let rejectReady!: (error: Error) => void; + const ready = new Promise((resolve, reject) => { + resolveReady = resolve; + rejectReady = reject; + }); + const state: NativeVideoExportFrameChannelState = { + sessionId, + port: channel.port1, + nextSequence: 0, + pending: new Map(), + ready, + resolveReady, + rejectReady, + readySettled: false, + closed: false, + handshakeTimeout: setTimeout(() => { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel handshake timed out", + ); + }, 2_000), + }; + nativeVideoExportFrameChannels.set(sessionId, state); + channel.port1.onmessage = (event: MessageEvent) => { + handleNativeVideoExportFramePortResponse(state, event.data); + }; + channel.port1.onmessageerror = () => { + settleNativeVideoExportFrameChannelState( + state, + "Native export frame channel delivery failed", + ); + }; + channel.port1.start(); + + try { + ipcRenderer.postMessage("native-video-export-frame-channel", { sessionId }, [ + channel.port2, + ]); + const capabilityProbe = new ArrayBuffer(1); + channel.port1.postMessage( + { + type: "hello", + protocol: 1, + sessionId, + capabilityProbe, + }, + [capabilityProbe], + ); + if (capabilityProbe.byteLength !== 0) { + settleNativeVideoExportFrameChannelState( + state, + "Native export transferable ArrayBuffer delivery is unavailable", + ); + } + } catch (error) { + settleNativeVideoExportFrameChannelState( + state, + error instanceof Error ? error.message : String(error), + ); + } + + return ready + .then(() => ({ success: true })) + .catch((error: unknown) => ({ + success: false, + error: error instanceof Error ? error.message : String(error), + fallbackAvailable: true, + })); +} + +function getNativeVideoExportTransferBuffer(frameData: Uint8Array): ArrayBuffer { + if ( + frameData.buffer instanceof ArrayBuffer && + frameData.byteOffset === 0 && + frameData.byteLength === frameData.buffer.byteLength + ) { + return frameData.buffer; + } + return frameData.slice().buffer as ArrayBuffer; +} + +async function writeNativeVideoExportFramesViaChannel( + sessionId: string, + frameDataList: Uint8Array[], +): Promise { + const state = nativeVideoExportFrameChannels.get(sessionId); + if (!state) { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + + try { + await state.ready; + } catch { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + + const acknowledgements: Array> = []; + let postedFrameCount = 0; + for (const frameData of frameDataList) { + const requestId = nextNativeVideoExportWriteRequestId++; + const sequence = state.nextSequence++; + const frame = getNativeVideoExportTransferBuffer(frameData); + const acknowledgement = new Promise((resolve) => { + state.pending.set(requestId, { sequence, resolve }); + }); + acknowledgements.push(acknowledgement); + try { + state.port.postMessage( + { + type: "frame", + protocol: 1, + sessionId, + requestId, + sequence, + frame, + }, + [frame], + ); + postedFrameCount += 1; + if (frame.byteLength !== 0) { + throw new Error("Native export transferable ArrayBuffer delivery is unavailable"); + } + } catch (error) { + state.pending.delete(requestId); + const message = error instanceof Error ? error.message : String(error); + settleNativeVideoExportFrameChannelState(state, message); + if (postedFrameCount === 0) { + return writeNativeVideoExportFramesLegacy(sessionId, frameDataList); + } + return { success: false, error: message, fallbackAvailable: false }; + } + } + + const results = await Promise.all(acknowledgements); + return results.find((result) => !result.success) ?? { success: true }; +} + +function closeNativeVideoExportFrameChannel(sessionId: string, error: string) { + const state = nativeVideoExportFrameChannels.get(sessionId); + if (state) { + settleNativeVideoExportFrameChannelState(state, error); + } +} + contextBridge.exposeInMainWorld("electronAPI", { hudOverlaySetIgnoreMouse: (ignore: boolean) => { ipcRenderer.send("hud-overlay-set-ignore-mouse", ignore); @@ -223,12 +579,27 @@ contextBridge.exposeInMainWorld("electronAPI", { nativeStaticLayoutExport: (options: { sessionId?: string; inputPath: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; width: number; height: number; frameRate: number; bitrate: number; encodingMode: "fast" | "balanced" | "quality"; durationSec: number; + overlayLayers?: Array<{ + id: string; + order: number; + path: string; + x: number; + y: number; + width: number; + height: number; + frameRate: number; + durationSec: number; + frameCount: number; + pixelFormat: "rgba"; + }>; contentWidth: number; contentHeight: number; offsetX: number; @@ -270,7 +641,15 @@ contextBridge.exposeInMainWorld("electronAPI", { anchorY: number; aspectRatio: number; }>; - zoomTelemetry?: Array<{ timeMs: number; scale: number; x: number; y: number }>; + zoomTelemetry?: Array<{ + timeMs: number; + scale: number; + x: number; + y: number; + blurStrength?: number; + blurCenterX?: number; + blurCenterY?: number; + }>; timelineSegments?: Array<{ sourceStartMs: number; sourceEndMs: number; @@ -297,6 +676,14 @@ contextBridge.exposeInMainWorld("electronAPI", { return ipcRenderer.invoke("native-static-layout-export", options) as Promise<{ success: boolean; tempPath?: string; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; + route?: + | "cuda-overlay" + | "cuda-scale-cpu-pad" + | "cuda-static-composite" + | "nvidia-cuda-compositor" + | "windows-d3d11-compositor"; encoderName?: string; error?: string; metrics?: NativeStaticLayoutMetrics; @@ -322,10 +709,21 @@ contextBridge.exposeInMainWorld("electronAPI", { bitrate: number; encodingMode: "fast" | "balanced" | "quality"; inputMode?: "rawvideo" | "h264-stream"; + videoCodec?: "h264" | "hevc"; + encoderPreference?: "auto" | "hardware" | "cpu"; }) => { return ipcRenderer.invoke("native-video-export-start", options); }, + nativeVideoExportOpenFrameChannel: (sessionId: string) => + openNativeVideoExportFrameChannel(sessionId), + nativeVideoExportWriteFrameViaChannel: (sessionId: string, frameData: Uint8Array) => + writeNativeVideoExportFramesViaChannel(sessionId, [frameData]), + nativeVideoExportWriteFramesViaChannel: (sessionId: string, frameDataList: Uint8Array[]) => + writeNativeVideoExportFramesViaChannel(sessionId, frameDataList), nativeVideoExportWriteFrame: (sessionId: string, frameData: Uint8Array) => { + if (nativeVideoExportFrameChannels.has(sessionId)) { + return writeNativeVideoExportFramesViaChannel(sessionId, [frameData]); + } ensureNativeVideoExportWriteResultListener(); return new Promise((resolve) => { @@ -343,6 +741,9 @@ contextBridge.exposeInMainWorld("electronAPI", { }); }, nativeVideoExportWriteFrames: (sessionId: string, frameDataList: Uint8Array[]) => { + if (nativeVideoExportFrameChannels.has(sessionId)) { + return writeNativeVideoExportFramesViaChannel(sessionId, frameDataList); + } ensureNativeVideoExportWriteResultListener(); return new Promise((resolve) => { @@ -390,7 +791,20 @@ contextBridge.exposeInMainWorld("electronAPI", { }, ); + closeNativeVideoExportFrameChannel( + sessionId, + result?.success + ? "Native video export session finished" + : "Native video export session failed", + ); return result; + }) + .catch((error: unknown) => { + closeNativeVideoExportFrameChannel( + sessionId, + "Native video export finish request failed", + ); + throw error; }) as Promise<{ success: boolean; data?: Uint8Array; @@ -401,6 +815,10 @@ contextBridge.exposeInMainWorld("electronAPI", { }, nativeVideoExportCancel: (sessionId: string) => { return ipcRenderer.invoke("native-video-export-cancel", sessionId).finally(() => { + closeNativeVideoExportFrameChannel( + sessionId, + "Native video export session was cancelled", + ); settleNativeVideoExportPendingRequests(sessionId, { success: false, error: "Native video export session was cancelled", @@ -765,7 +1183,7 @@ contextBridge.exposeInMainWorld("electronAPI", { }, getLocalMediaUrl: (filePath: string) => { return ipcRenderer.invoke("get-local-media-url", filePath) as Promise< - { success: true; url: string } | { success: false } + { success: true; url: string; pending?: boolean } | { success: false } >; }, saveProjectFile: ( diff --git a/scripts/benchmark-cuda4k.mjs b/scripts/benchmark-cuda4k.mjs new file mode 100644 index 000000000..a225a85da --- /dev/null +++ b/scripts/benchmark-cuda4k.mjs @@ -0,0 +1,188 @@ +// 4K CUDA compositor benchmark runner. Generates nothing; expects a prepared +// .tmp/cuda4k workspace with source-4k.mp4, overlay-4k.rgba, overlay-manifest.json, +// cursor-telemetry.json, zoom-telemetry.csv. +// +// Usage: node scripts/benchmark-cuda4k.mjs [--tag name] [--frames N] [--temporal N] [--overlay 0|1] [--cursor 0|1] [--codec h264|hevc] +import { execFileSync, spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, ".."); +const workDir = path.join(repoRoot, ".tmp", "cuda4k"); +const pipeline = path.join( + repoRoot, + "electron", + "native", + "nvidia-cuda-compositor", + "run-mp4-pipeline.mjs", +); +const ffmpeg = path.join(repoRoot, "node_modules", "ffmpeg-static", "ffmpeg.exe"); +const ffprobe = path.join( + repoRoot, + "node_modules", + "ffprobe-static", + "bin", + "win32", + "x64", + "ffprobe.exe", +); + +function arg(name, fallback) { + const index = process.argv.indexOf(name); + return index >= 0 && index + 1 < process.argv.length ? process.argv[index + 1] : fallback; +} + +const tag = arg("--tag", "baseline"); +const durationSec = Number(arg("--duration", "6")); +const fps = Number(arg("--fps", "30")); +const temporal = Number(arg("--temporal", "13")); +const withOverlay = arg("--overlay", "1") === "1"; +const withCursor = arg("--cursor", "0") === "1"; +const codec = arg("--codec", "hevc"); +const encodingMode = arg("--mode", "balanced"); + +if (!existsSync(path.join(workDir, "source-4k.mp4"))) { + console.error("Missing 4K source; run generation first"); + process.exit(2); +} + +const targetFrames = Math.ceil(durationSec * fps); +const outputPath = path.join(workDir, `out-${tag}-${codec}.mp4`); +const args = [ + pipeline, + "--input", + path.join(workDir, "source-4k.mp4"), + "--output", + outputPath, + "--output-codec", + codec, + "--width", + "3840", + "--height", + "2160", + "--fps", + String(fps), + "--bitrate-mbps", + "40", + "--encoding-mode", + encodingMode, + "--duration-sec", + String(durationSec), + "--stream-sync", + "--prewarm-ms", + "300", + "--content-x", + "0", + "--content-y", + "0", + "--content-width", + "3840", + "--content-height", + "2160", + "--radius", + "0", + "--background-y", + "16", + "--background-u", + "128", + "--background-v", + "128", + "--zoom-telemetry", + path.join(workDir, "zoom-telemetry.csv"), +]; +if (withOverlay) { + const manifestPath = path.join(workDir, "overlay-manifest.json"); + const absoluteManifest = { + layers: [ + { + id: "bench-overlay", + path: path.join(workDir, "overlay-4k.rgba"), + x: 800, + y: 500, + width: 900, + height: 560, + frameCount: targetFrames, + }, + ], + }; + writeFileSync(manifestPath, JSON.stringify(absoluteManifest)); + args.push("--overlay-manifest", manifestPath); +} +if (withCursor) { + args.push( + "--cursor-json", + path.join(workDir, "cursor-telemetry.json"), + "--cursor-height", + "84", + ); +} +if (temporal >= 3) { + args.push( + "--temporal-blur-sample-count", + String(temporal), + "--temporal-blur-shutter-fraction", + "0.5", + "--temporal-blur-weight-power", + "2", + ); +} + +const env = { + ...process.env, + RECORDLY_FFMPEG_EXE: ffmpeg, + RECORDLY_FFPROBE_EXE: ffprobe, + RECORDLY_NVIDIA_CUDA_EXPORT_HIGH_PRIORITY: "1", +}; + +const startedAt = performance.now(); +const result = spawnSync("node", args, { env, encoding: "utf8", maxBuffer: 512 * 1024 * 1024 }); +const elapsedMs = performance.now() - startedAt; +if (result.status !== 0) { + console.error("Benchmark run failed", result.status); + console.error(result.stderr.slice(-4000)); + process.exit(1); +} + +const stdout = result.stdout; +const lines = stdout.split(/\r?\n/); +const summaryStart = lines.findIndex((line) => line.trim() === "{"); +if (summaryStart === -1) { + console.error("No summary JSON found"); + process.exit(1); +} +const summary = JSON.parse(lines.slice(summaryStart).join("\n")); +const ns = summary.nativeSummary ?? {}; +const out = { + tag, + codec: summary.outputCodec, + temporal, + withOverlay: Boolean(ns.overlayLayers), + withCursor: Boolean(ns.cursorOverlay), + frames: ns.frames, + targetFrames: summary.targetFrames, + measuredFps: ns.measuredFps, + realtimeMultiplier: ns.realtimeMultiplier, + totalMs: ns.totalMs, + decodeMs: ns.decodeMs, + encodeMs: ns.encodeMs, + compositeMs: ns.compositeMs, + compositeGpuMs: ns.compositeGpuMs, + zoomBlurGpuMs: ns.zoomBlurGpuMs, + overlayBlendGpuMs: ns.overlayBlendGpuMs, + overlayUploadMs: ns.overlayUploadMs, + nvencMs: ns.nvencMs, + packetWriteMs: ns.packetWriteMs, + flushMs: ns.flushMs, + temporalBlurFrames: ns.temporalBlurFrames, + temporalBlurSamplesTotal: ns.temporalBlurSamplesTotal, + roiCompositeFrames: ns.roiCompositeFrames, + monolithicCompositeFrames: ns.monolithicCompositeFrames, + copyCompositeFrames: ns.copyCompositeFrames, + rcMode: ns.nvencDiagnostics?.rcModeUsed, + outputBytes: ns.outputBytes, + wallMs: Number(elapsedMs.toFixed(2)), +}; +console.log(JSON.stringify(out, null, 1)); +writeFileSync(path.join(workDir, `result-${tag}.json`), JSON.stringify(out, null, 1) + "\n"); diff --git a/scripts/build-nvidia-cuda-compositor.mjs b/scripts/build-nvidia-cuda-compositor.mjs index f1447c4b7..2d3e94dd8 100644 --- a/scripts/build-nvidia-cuda-compositor.mjs +++ b/scripts/build-nvidia-cuda-compositor.mjs @@ -1,5 +1,13 @@ import { execSync } from "node:child_process"; -import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + copyFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import path from "node:path"; import { @@ -28,6 +36,15 @@ const generatorArch = process.arch === "arm64" ? "ARM64" : "x64"; const videoCodecSdkRoot = process.env.RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT?.trim() || path.join(projectRoot, ".tmp", "video-sdk-samples"); +const nvEncHeadersRoot = + process.env.RECORDLY_NVENC_HEADERS_ROOT?.trim() || + path.join(projectRoot, ".tmp", "nv-codec-headers"); +// nvEncodeAPI.h 13.x is required for Blackwell-era NVENC; the public +// Video Codec SDK samples repo still ships the legacy 8.1 header which fails +// with NV_ENC_ERR_INVALID_PARAM (error 8) on current drivers. Pin the FFmpeg +// nv-codec-headers release that provides API 13.0. +const NVENC_HEADERS_TAG = "n13.0.19.1"; +const NVENC_HEADERS_INCLUDE = path.join(nvEncHeadersRoot, "include", "ffnvcodec"); if (process.platform !== "win32") { console.log("[build-nvidia-cuda-compositor] Skipping NVIDIA CUDA compositor build."); @@ -115,6 +132,68 @@ function findCmake() { return null; } +function findCudaToolkitRoot() { + const candidates = [ + process.env.CUDA_PATH, + ...Object.entries(process.env) + .filter(([name]) => /^CUDA_PATH_V\d+_\d+$/.test(name)) + .map(([, value]) => value), + ]; + const cudaInstallRoot = path.join( + "C:", + "Program Files", + "NVIDIA GPU Computing Toolkit", + "CUDA", + ); + if (existsSync(cudaInstallRoot)) { + candidates.push( + ...readdirSync(cudaInstallRoot) + .sort() + .reverse() + .map((version) => path.join(cudaInstallRoot, version)), + ); + } + + return ( + candidates + .filter((candidate) => typeof candidate === "string" && candidate.length > 0) + .map((candidate) => path.normalize(candidate)) + .find((candidate) => existsSync(path.join(candidate, "bin", "nvcc.exe"))) ?? null + ); +} + +function ensureNvEncHeaders() { + if (existsSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"))) { + return; + } + console.log(`[build-nvidia-cuda-compositor] Cloning nv-codec-headers ${NVENC_HEADERS_TAG}...`); + execSync( + `git clone --depth 1 --branch ${NVENC_HEADERS_TAG} https://github.com/FFmpeg/nv-codec-headers.git "${nvEncHeadersRoot}"`, + { stdio: "inherit" }, + ); + if (!existsSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"))) { + fallbackToBundledHelperOrExit( + `nv-codec-headers ${NVENC_HEADERS_TAG} could not be staged; a Blackwell-compatible nvEncodeAPI.h is required.`, + ); + } + // The legacy samples checkout ships nvEncodeAPI.h 8.1; the compiler picks the + // quoted include from the NvEncoder directory first, so the 13.0 header must + // replace it to build the encoder library against the current API. + const samplesHeader = path.join( + videoCodecSdkRoot, + "Samples", + "NvCodec", + "NvEncoder", + "nvEncodeAPI.h", + ); + const versionLine = readFileSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"), "utf8") + .split(/\r?\n/) + .find((line) => line.includes("NVENCAPI_MAJOR_VERSION")); + if (existsSync(samplesHeader) && !/NVENCAPI_MAJOR_VERSION 13/.test(versionLine ?? "")) { + copyFileSync(path.join(NVENC_HEADERS_INCLUDE, "nvEncodeAPI.h"), samplesHeader); + } +} + if (!existsSync(path.join(videoCodecSdkRoot, "Samples", "NvCodec"))) { fallbackToBundledHelperOrExit( `NVIDIA Video Codec SDK samples not found at ${videoCodecSdkRoot}. Set RECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT to build from source.`, @@ -217,6 +296,69 @@ using RecordlyDisplayFramePolicy = bool (*)(int, void*); } } +try { + ensureNvEncHeaders(); +} catch (error) { + fallbackToBundledHelperOrExit( + `Failed to stage nv-codec-headers: ${error instanceof Error ? error.message : String(error)}`, + ); +} + +// The samples NvEncoder library predates nvEncodeAPI 13.x; patch the few API +// incompatibilities so it compiles against the staged header. +function patchNvEncoderForNvEnc13Headers() { + const nvEncoderDir = path.join(videoCodecSdkRoot, "Samples", "NvCodec", "NvEncoder"); + const sourcePath = path.join(nvEncoderDir, "NvEncoder.cpp"); + let source = readFileSync(sourcePath, "utf8"); + + if (!source.includes("nvEncEncodePicture API failed: ")) { + source = replaceOrThrow( + sourcePath, + source, + /if \(pIntializeParams->presetGUID != NV_ENC_PRESET_LOSSLESS_DEFAULT_GUID\r?\n(?:\s+)&& pIntializeParams->presetGUID != NV_ENC_PRESET_LOSSLESS_HP_GUID\)\r?\n(?:\s+)\{\r?\n(?:\s+)pIntializeParams->encodeConfig->rcParams\.constQP = \{ 28, 31, 25 \};\r?\n(?:\s+)\}/, + " pIntializeParams->encodeConfig->rcParams.constQP = { 28, 31, 25 };", + "NVENC 13 lossless preset GUID check", + ); + source = replaceOrThrow( + sourcePath, + source, + /pIntializeParams->encodeConfig->encodeCodecConfig\.hevcConfig\.pixelBitDepthMinus8 =\r?\n(?:\s+)\(m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV420_10BIT \|\| m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV444_10BIT \) \? 2 : 0;\r?\n/, + "", + "NVENC 13 HEVC bit depth field removal", + ); + source = replaceOrThrow( + sourcePath, + source, + /bool yuv10BitFormat = \(m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV420_10BIT \|\| m_eBufferFormat == NV_ENC_BUFFER_FORMAT_YUV444_10BIT\) \? true : false;\r?\n(?:\s+)if \(yuv10BitFormat && pEncoderParams->encodeConfig->encodeCodecConfig\.hevcConfig\.pixelBitDepthMinus8 != 2\)\r?\n(?:\s+)\{\r?\n(?:\s+)NVENC_THROW_ERROR\("Invalid PixelBitdepth", NV_ENC_ERR_INVALID_PARAM\);\r?\n(?:\s+)\}\r?\n\r?\n/, + "", + "NVENC 13 HEVC bit depth check removal", + ); + source = replaceOrThrow( + sourcePath, + source, + /NV_ENC_PRESET_DEFAULT_GUID/, + "NV_ENC_PRESET_P4_GUID", + "NVENC 13 default preset GUID", + ); + source = replaceOrThrow( + sourcePath, + source, + /"nvEncEncodePicture API failed"/, + '"nvEncEncodePicture API failed: " + std::to_string(nvStatus)', + "NVENC encode-picture error detail", + ); + writeFileSync(sourcePath, source); + } +} + +try { + patchNvEncoderForNvEnc13Headers(); +} catch (error) { + fallbackToBundledHelperOrExit( + `Failed to patch NVIDIA NvEncoder for NVENC 13 headers: ${error instanceof Error ? error.message : String(error)}`, + ); +} + try { patchNvDecoderForRecordlyCallbacks(); } catch (error) { @@ -232,6 +374,13 @@ if (!cmake) { ); } +const cudaToolkitRoot = findCudaToolkitRoot(); +if (!cudaToolkitRoot) { + fallbackToBundledHelperOrExit( + "CUDA Toolkit not found. Install CUDA Toolkit or set CUDA_PATH before building.", + ); +} + mkdirSync(buildDir, { recursive: true }); function clearCmakeCache() { @@ -246,7 +395,15 @@ try { clearCache: clearCmakeCache, configure: (generator, toolset) => execSync( - `${cmake} .. -G "${generator}" -A ${generatorArch}${toolset ? ` -T ${toolset}` : ""} -DRECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT="${videoCodecSdkRoot}"`, + `${cmake} .. -G "${generator}" -A ${generatorArch} -T "${[ + toolset, + `cuda=${cudaToolkitRoot}`, + "host=x64", + ] + .filter(Boolean) + .join( + ",", + )}" -DCMAKE_CUDA_COMPILER="${path.join(cudaToolkitRoot, "bin", "nvcc.exe")}" -DCUDAToolkit_ROOT="${cudaToolkitRoot}" -DRECORDLY_NVIDIA_VIDEO_CODEC_SDK_ROOT="${videoCodecSdkRoot}"`, { cwd: buildDir, stdio: "inherit", diff --git a/src/components/video-editor/ExportSettingsMenu.tsx b/src/components/video-editor/ExportSettingsMenu.tsx index 5e36cdd76..a09adf777 100644 --- a/src/components/video-editor/ExportSettingsMenu.tsx +++ b/src/components/video-editor/ExportSettingsMenu.tsx @@ -1,18 +1,30 @@ import { DownloadSimple as Download, FilmSlate as Film, Image } from "@phosphor-icons/react"; import { LayoutGroup, motion } from "motion/react"; +import { useEffect, useState } from "react"; import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; import { Switch } from "@/components/ui/switch"; import { useScopedT } from "@/contexts/I18nContext"; import type { + ExportBitrateMode, + ExportEncoderPreference, ExportEncodingMode, ExportFormat, ExportMp4FrameRate, ExportPipelineModel, ExportQuality, + ExportVideoCodec, GifFrameRate, GifSizePreset, } from "@/lib/exporter"; -import { GIF_FRAME_RATES, GIF_SIZE_PRESETS, MP4_FRAME_RATES } from "@/lib/exporter"; +import { + EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_MAX_MBPS, + EXPORT_BITRATE_MIN_MBPS, + GIF_FRAME_RATES, + GIF_SIZE_PRESETS, + MP4_FRAME_RATES, +} from "@/lib/exporter"; import { cn } from "@/lib/utils"; interface ExportSettingsMenuProps { @@ -22,6 +34,14 @@ interface ExportSettingsMenuProps { onExportQualityChange?: (quality: ExportQuality) => void; exportEncodingMode: ExportEncodingMode; onExportEncodingModeChange?: (encodingMode: ExportEncodingMode) => void; + exportVideoCodec: ExportVideoCodec; + onExportVideoCodecChange?: (codec: ExportVideoCodec) => void; + exportEncoderPreference: ExportEncoderPreference; + onExportEncoderPreferenceChange?: (preference: ExportEncoderPreference) => void; + exportBitrateMode: ExportBitrateMode; + onExportBitrateModeChange?: (mode: ExportBitrateMode) => void; + exportBitrateMbps?: number; + onExportBitrateMbpsChange?: (mbps: number) => void; mp4FrameRate: ExportMp4FrameRate; onMp4FrameRateChange?: (frameRate: ExportMp4FrameRate) => void; exportPipelineModel?: ExportPipelineModel; @@ -29,6 +49,11 @@ interface ExportSettingsMenuProps { experimentalNvidiaCudaExport?: boolean; onExperimentalNvidiaCudaExportChange?: (enabled: boolean) => void; nvidiaCudaExportAvailable?: boolean; + nvidiaCudaExportSkipReason?: string | null; + /** True when HEVC + Hardware makes the NVIDIA CUDA compositor mandatory. The + * option is shown as selected and cannot be disabled; the export hard-fails + * instead of falling back when the compositor cannot run. */ + nvidiaCudaCompositorRequired?: boolean; showCaptionSidecarOption?: boolean; includeCaptionSidecar?: boolean; onIncludeCaptionSidecarChange?: (enabled: boolean) => void; @@ -51,6 +76,14 @@ export function ExportSettingsMenu({ onExportQualityChange, exportEncodingMode, onExportEncodingModeChange, + exportVideoCodec, + onExportVideoCodecChange, + exportEncoderPreference, + onExportEncoderPreferenceChange, + exportBitrateMode, + onExportBitrateModeChange, + exportBitrateMbps = EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + onExportBitrateMbpsChange, mp4FrameRate, onMp4FrameRateChange, exportPipelineModel = "modern", @@ -58,6 +91,8 @@ export function ExportSettingsMenu({ experimentalNvidiaCudaExport = false, onExperimentalNvidiaCudaExportChange, nvidiaCudaExportAvailable = false, + nvidiaCudaExportSkipReason = null, + nvidiaCudaCompositorRequired = false, showCaptionSidecarOption = false, includeCaptionSidecar = false, onIncludeCaptionSidecarChange, @@ -74,6 +109,20 @@ export function ExportSettingsMenu({ }: ExportSettingsMenuProps) { const tSettings = useScopedT("settings"); const isLegacyModel = exportPipelineModel === "legacy"; + const [bitrateDraft, setBitrateDraft] = useState(String(exportBitrateMbps)); + + useEffect(() => { + setBitrateDraft(String(exportBitrateMbps)); + }, [exportBitrateMbps]); + + const commitBitrateDraft = () => { + const parsed = Number(bitrateDraft); + const clamped = Number.isFinite(parsed) + ? Math.min(EXPORT_BITRATE_MAX_MBPS, Math.max(EXPORT_BITRATE_MIN_MBPS, parsed)) + : EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS; + setBitrateDraft(String(clamped)); + onExportBitrateMbpsChange?.(clamped); + }; return (
- {option.label} - - {mp4OutputDimensions ? ( - + {option.label} + + {mp4OutputDimensions ? ( + {mp4OutputDimensions[option.value].width} x{" "} @@ -216,7 +267,155 @@ export function ExportSettingsMenu({ {isActive ? ( + ) : null} + + {option.label} + + + ); + })} +
+
+ + {tSettings("export.codecTitle", "Video codec")} + +
+
+ {( + [ + { value: "h264", label: tSettings("export.codec.h264", "H.264") }, + { value: "hevc", label: tSettings("export.codec.hevc", "H.265") }, + ] as const + ).map((option) => { + const isActive = exportVideoCodec === option.value; + return ( + + ); + })} +
+
+ + {tSettings("export.encoderTitle", "Encoder")} + +
+
+ {( + [ + { value: "auto", label: tSettings("export.encoder.auto", "Auto") }, + { + value: "hardware", + label: tSettings("export.encoder.hardware", "Hardware"), + }, + { value: "cpu", label: tSettings("export.encoder.cpu", "CPU") }, + ] as const + ).map((option) => { + const isActive = exportEncoderPreference === option.value; + return ( + + ); + })} +
+
+ + {tSettings("export.bitrateTitle", "Bitrate")} + +
+
+ {( + [ + { + value: "auto", + label: tSettings("export.bitrate.auto", "Auto"), + }, + { + value: "custom", + label: tSettings("export.bitrate.custom", "Custom"), + }, + ] as const + ).map((option) => { + const isActive = exportBitrateMode === option.value; + return ( +
- + {tSettings("export.codecTitle", "Video codec")} + + +
@@ -439,11 +449,27 @@ export function ExportSettingsMenu({
{exportBitrateMode === "custom" ? (
+
+ { + setBitrateDraft(String(value)); + onExportBitrateMbpsChange?.(value); + }} + aria-label={tSettings( + "export.bitrate.mbpsInput", + "Custom bitrate in Mbps", + )} + /> +
{ setBitrateDraft(event.target.value); @@ -453,26 +479,15 @@ export function ExportSettingsMenu({ } }} onBlur={commitBitrateDraft} - className="h-8 w-24" + className="h-8 w-14 text-center text-[11px] border-foreground/10 bg-foreground/[0.03] text-foreground" aria-label={tSettings( "export.bitrate.mbpsInput", "Custom bitrate in Mbps", )} /> Mbps - - {tSettings("export.bitrate.range", "1\u2013200 Mbps")} -
) : null} - {exportVideoCodec === "hevc" ? ( -

- {tSettings( - "export.hevcHint", - "HEVC (H.265) makes smaller files but may not play on older or web players. Recordly preview playback is unchanged.", - )} -

- ) : null}
{tSettings("export.fpsTitle", "FPS")} @@ -514,183 +529,117 @@ export function ExportSettingsMenu({ ); })}
-
- - {tSettings("export.pipelineTitle", "Pipeline")} - -
-
- {( - [ - { - value: "legacy", - label: tSettings("export.pipeline.legacy", "Legacy"), - }, - { - value: "modern", - label: tSettings("export.pipeline.modern", "Lightning (Beta)"), - }, - ] as const - ).map((option) => { - const isActive = exportPipelineModel === option.value; - return ( - - ); - })} -
-

- {isLegacyModel - ? tSettings( - "export.pipeline.legacyHint", - "Legacy uses the current stable WebCodecs export path.", - ) - : tSettings( - "export.pipeline.lightningHint", - "Lightning (Beta) automatically uses the fastest compatible backend and falls back when needed.", - )} -

- {!isLegacyModel ? ( -
-
-
-
- + {nvidiaCudaCompositorRequired ? ( + {tSettings( - "export.nvidiaCuda.compositorTitle", - "NVIDIA CUDA compositor", + "export.nvidiaCuda.requiredBadge", + "Required", )} - {nvidiaCudaCompositorRequired ? ( - - {tSettings( - "export.nvidiaCuda.requiredBadge", - "Required", - )} - - ) : experimentalNvidiaCudaExport ? ( - - {tSettings( - "export.nvidiaCuda.selectedBadge", - "Selected", - )} - - ) : nvidiaCudaExportAvailable ? ( - - {tSettings( - "export.nvidiaCuda.availableBadge", - "Available", - )} - - ) : ( - - {tSettings( - "export.nvidiaCuda.unavailableBadge", - "Unavailable", - )} - - )} -
-

- {tSettings("export.nvidiaCuda.backendLabel", "Backend")} - {": "} - {nvidiaCudaCompositorRequired || - experimentalNvidiaCudaExport - ? tSettings( - "export.nvidiaCuda.backendSelected", - "NVIDIA CUDA compositor", - ) - : tSettings("export.backend.auto", "Auto")} -

-

- {nvidiaCudaCompositorRequired + ) : experimentalNvidiaCudaExport ? ( + + {tSettings( + "export.nvidiaCuda.selectedBadge", + "Selected", + )} + + ) : nvidiaCudaExportAvailable ? ( + + {tSettings( + "export.nvidiaCuda.availableBadge", + "Available", + )} + + ) : ( + + {tSettings( + "export.nvidiaCuda.unavailableBadge", + "Unavailable", + )} + + )} +

+

+ {tSettings("export.nvidiaCuda.backendLabel", "Backend")} + {": "} + {nvidiaCudaCompositorRequired || experimentalNvidiaCudaExport + ? tSettings( + "export.nvidiaCuda.backendSelected", + "NVIDIA CUDA compositor", + ) + : tSettings("export.backend.auto", "Auto")} +

+

+ {nvidiaCudaCompositorRequired + ? tSettings( + "export.nvidiaCuda.hintRequired", + "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + ) + : experimentalNvidiaCudaExport ? tSettings( - "export.nvidiaCuda.hintRequired", - "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", + "export.nvidiaCuda.hintSelected", + "Exports will use the NVIDIA CUDA compositor on this device.", ) - : experimentalNvidiaCudaExport + : nvidiaCudaExportAvailable ? tSettings( - "export.nvidiaCuda.hintSelected", - "Exports will use the NVIDIA CUDA compositor on this device.", + "export.nvidiaCuda.hint", + "Compose and encode on the NVIDIA GPU for fast exports.", ) - : nvidiaCudaExportAvailable + : nvidiaCudaExportSkipReason ? tSettings( - "export.nvidiaCuda.hint", - "Compose and encode on the NVIDIA GPU for fast exports.", + "export.nvidiaCuda.unavailableReason", + `CUDA compositor is unavailable (${nvidiaCudaExportSkipReason}).`, + { + reason: nvidiaCudaExportSkipReason, + }, ) - : nvidiaCudaExportSkipReason - ? tSettings( - "export.nvidiaCuda.unavailableReason", - `CUDA compositor is unavailable (${nvidiaCudaExportSkipReason}).`, - { - reason: nvidiaCudaExportSkipReason, - }, - ) - : tSettings( - "export.nvidiaCuda.unavailableGeneric", - "CUDA compositor is unavailable on this device.", - )} -

- {nvidiaCudaCompositorRequired && !nvidiaCudaExportAvailable ? ( -

- {tSettings( - "export.nvidiaCuda.unavailableRequired", - "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto.", - )} -

- ) : null} -
- {nvidiaCudaCompositorRequired ? ( - - ) : nvidiaCudaExportAvailable ? ( - + {nvidiaCudaCompositorRequired && !nvidiaCudaExportAvailable ? ( +

+ {tSettings( + "export.nvidiaCuda.unavailableRequired", + "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto.", )} - className="shrink-0 scale-75 data-[state=checked]:bg-[#2563EB]" - /> +

) : null}
+ {nvidiaCudaCompositorRequired ? ( + + ) : nvidiaCudaExportAvailable ? ( + + ) : null} - ) : null} + {showCaptionSidecarOption ? (
diff --git a/src/components/video-editor/mp4ExportRouting.test.ts b/src/components/video-editor/mp4ExportRouting.test.ts index c304a7e9d..300dbbe3a 100644 --- a/src/components/video-editor/mp4ExportRouting.test.ts +++ b/src/components/video-editor/mp4ExportRouting.test.ts @@ -199,6 +199,19 @@ describe("resolveMp4ExportRouting", () => { expect(result.backendPreference).toBe("auto"); }); + it("routes H.264 Hardware to the native GPU compositor when RTX Rendering is on", () => { + const result = resolveMp4ExportRouting({ + ...baseOptions, + exportVideoCodec: "h264", + exportEncoderPreference: "hardware", + experimentalNvidiaCudaExport: true, + nvidiaCudaExportAvailable: true, + }); + expect(result.useExperimentalNvidiaCudaExport).toBe(true); + expect(result.needsNativeRawFrame).toBe(false); + expect(result.pipelineModel).toBe("modern"); + }); + it("keeps the existing legacy/auto route for H.264 with auto encoder preference", () => { const legacy = resolveMp4ExportRouting({ ...baseOptions, diff --git a/src/components/video-editor/mp4ExportRouting.ts b/src/components/video-editor/mp4ExportRouting.ts index 1cff4ca2a..a474d2a31 100644 --- a/src/components/video-editor/mp4ExportRouting.ts +++ b/src/components/video-editor/mp4ExportRouting.ts @@ -53,9 +53,11 @@ export function resolveMp4ExportRouting({ const useExperimentalNativeExport = pipelineModel === "modern" && (smokeExportConfig.enabled ? smokeExportConfig.useNativeExport : true); + // Auto and explicit GPU/Hardware preferences may use the native NVIDIA + // compositor for BOTH codecs (H.264 and HEVC). CPU stays on the software + // encoder; the H.264 compatibility default (Auto) is unchanged. const mayUseNativeGpuCompositor = - exportEncoderPreference === "auto" || - (exportVideoCodec === "hevc" && exportEncoderPreference === "hardware"); + exportEncoderPreference === "auto" || exportEncoderPreference === "hardware"; // HEVC + Hardware makes the NVIDIA CUDA compositor mandatory: the user's codec // and encoder choice IS the opt-in, so the CUDA route no longer depends on a // hidden experimental toggle. If the compositor cannot run (no helper, no @@ -78,10 +80,15 @@ export function resolveMp4ExportRouting({ exportVideoCodec === "hevc" && exportEncoderPreference !== "cpu" && useExperimentalNvidiaCudaExport; + // H.264 Auto stays on the compatibility path (native layout with the + // automatic bitrate heuristic, WebCodecs/native/Breeze routing unchanged). + // H.264 Hardware uses the native GPU compositor when eligible, exactly like + // HEVC Hardware; only H.264 CPU forces the raw software frame path. const needsNativeRawFrame = exportVideoCodec === "hevc" ? !canUseHevcNativeGpuCompositor - : exportEncoderPreference !== "auto"; + : exportEncoderPreference === "cpu" || + (exportEncoderPreference === "hardware" && !useExperimentalNvidiaCudaExport); const backendPreference = pipelineModel === "legacy" ? "webcodecs" diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 4eaf2ac12..0512feeb2 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -240,28 +240,28 @@ "encoderTitle": "Encoder", "encoder": { "auto": "Auto", - "hardware": "Hardware", + "hardware": "GPU", "cpu": "CPU" }, "backend": { "auto": "Auto" }, "nvidiaCuda": { - "compositorTitle": "NVIDIA CUDA compositor", + "compositorTitle": "NVIDIA RTX Rendering", "backendLabel": "Backend", - "backendSelected": "NVIDIA CUDA compositor", + "backendSelected": "NVIDIA RTX Rendering", "availableBadge": "Available", "selectedBadge": "Selected", "requiredBadge": "Required", "unavailableBadge": "Unavailable", - "toggle": "Use the NVIDIA CUDA compositor for GPU-accelerated exports", - "requiredToggle": "NVIDIA CUDA compositor is required for H.265 Hardware exports", + "toggle": "Use NVIDIA RTX Rendering for GPU-accelerated exports", + "requiredToggle": "NVIDIA RTX Rendering is required for H.265 GPU exports", "hint": "Compose and encode on the NVIDIA GPU for fast exports.", - "hintSelected": "Exports will use the NVIDIA CUDA compositor on this device.", - "hintRequired": "H.265 Hardware exports use the NVIDIA CUDA compositor and never fall back to renderer frames.", - "unavailableReason": "CUDA compositor is unavailable ({{reason}}).", - "unavailableGeneric": "CUDA compositor is unavailable on this device.", - "unavailableRequired": "H.265 + Hardware exports will fail until the CUDA compositor is available. Install or update NVIDIA drivers, or switch Encoder to Auto." + "hintSelected": "Exports will use NVIDIA RTX Rendering on this device.", + "hintRequired": "H.265 GPU exports use NVIDIA RTX Rendering and never fall back to renderer frames.", + "unavailableReason": "RTX Rendering is unavailable ({{reason}}).", + "unavailableGeneric": "RTX Rendering is unavailable on this device.", + "unavailableRequired": "H.265 + GPU exports will fail until RTX Rendering is available. Install or update NVIDIA drivers, or switch Encoder to Auto." }, "bitrateTitle": "Bitrate", "bitrate": { @@ -270,7 +270,6 @@ "mbpsInput": "Custom bitrate in Mbps", "range": "1–200 Mbps" }, - "hevcHint": "HEVC (H.265) makes smaller files but may not play on older or web players. Recordly preview playback is unchanged.", "hardwareUnavailable": "No usable hardware encoder was found. Pick CPU or Auto, or update your GPU driver.", "errors": { "hardwareUnavailable": "No usable hardware encoder was found. Pick CPU or Auto, or update your GPU driver." From c8ee9d6eb489dd363f3bc2f67c6b6978eb48c02e Mon Sep 17 00:00:00 2001 From: nmzpy <246990748+nmzpy@users.noreply.github.com> Date: Thu, 6 Aug 2026 18:34:57 +0300 Subject: [PATCH 09/11] fix(export): address CodeRabbit review findings - strict HEVC Hardware hard-fails (noCpuFallback) instead of rawvideo fallback in the route planner - enforce codec-specific custom bitrate caps (H.264 105 / HEVC 70 Mbps) in resolve/clamp/persistence - reject temporal blur plans below the CUDA minimum sample count instead of dropping the effect - bound completed frame request ids with a monotonic watermark - discard produced native temp video on HEVC route rejection - deterministic precomposited overlay z-order - clamp bitrate input on change; update tests and codec range strings --- electron/ipc/export/native-video.test.ts | 1141 +++++++++++++++- electron/ipc/export/native-video.ts | 1195 ++++++++++++++--- .../nativeStaticLayoutRoutePlan.test.ts | 25 +- .../ipc/export/nativeStaticLayoutRoutePlan.ts | 20 +- electron/ipc/nativeVideoExport.test.ts | 69 + electron/ipc/nativeVideoExport.ts | 7 +- electron/ipc/register/export.ts | 84 +- .../run-mp4-pipeline.mjs | 78 +- .../video-editor/ExportSettingsMenu.tsx | 7 +- .../video-editor/editorPreferences.test.ts | 6 +- .../video-editor/projectPersistence.test.ts | 9 +- .../video-editor/projectPersistence.ts | 12 +- src/i18n/locales/de/settings.json | 2 +- src/i18n/locales/en/settings.json | 2 +- src/i18n/locales/es/settings.json | 2 +- src/i18n/locales/fr/settings.json | 2 +- src/i18n/locales/it/settings.json | 2 +- src/i18n/locales/ko/settings.json | 2 +- src/lib/exporter/exportBitrate.test.ts | 21 +- src/lib/exporter/exportBitrate.ts | 27 +- ...rnVideoExporter.nativeStaticLayout.test.ts | 11 + ...rnVideoExporter.overlayPreparation.test.ts | 908 ++++++++++++- src/lib/exporter/modernVideoExporter.ts | 910 ++++++++++++- src/lib/exporter/types.ts | 2 + 24 files changed, 4269 insertions(+), 275 deletions(-) diff --git a/electron/ipc/export/native-video.test.ts b/electron/ipc/export/native-video.test.ts index c47d16e28..234a94bb2 100644 --- a/electron/ipc/export/native-video.test.ts +++ b/electron/ipc/export/native-video.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; vi.mock("electron", () => ({ app: { @@ -26,7 +26,11 @@ const fsMocks = vi.hoisted(() => ({ writeFile: vi.fn(async () => undefined), readFile: vi.fn(), stat: vi.fn(async () => ({ size: 5_000_000_000 })), + realpath: vi.fn(async (pathValue: string) => pathValue), unlink: vi.fn(async () => undefined), + mkdir: vi.fn(async () => undefined), + rm: vi.fn(async () => undefined), + copyFile: vi.fn(async () => undefined), })); vi.mock("node:fs/promises", () => ({ @@ -46,6 +50,8 @@ vi.mock("node:child_process", () => ({ spawn: vi.fn(), })); +import { spawn } from "node:child_process"; +import { EventEmitter } from "node:events"; import { app } from "electron"; import type { NativeTiledOverlayLayerDescriptor } from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; import { @@ -56,7 +62,10 @@ import { buildNativeStaticLayoutTiledOverlayManifest, buildNativeStaticLayoutTimelineSegments, buildNativeVideoAudioMuxArgs, + buildNvidiaCudaPrepareProgress, canCopyAudioCodecIntoMp4, + cancelInFlightCapabilityOnlyPrewarms, + canReuseNativeStaticLayoutSourceProbe, exportNativeStaticLayoutVideo, formatNativeStaticLayoutZoomTelemetryLines, getExperimentalNvidiaCudaExportSkipReason, @@ -71,6 +80,7 @@ import { hasNvidiaGpuDeviceInGpuInfo, mapNvidiaCudaWrapperProgressPercentage, muxExportedVideoAudioBuffer, + muxNativeVideoExportAudio, type NativeStaticLayoutExportOptions, type NativeStaticLayoutOverlayLayer, normalizeNativeStaticLayoutBackground, @@ -82,12 +92,17 @@ import { parseNvidiaCudaExportSummary, parseWindowsGpuExportProgressLine, parseWindowsGpuExportSummary, + prewarmNativeExportCaches, + registerCapabilityOnlyPrewarmChild, + resetNativeStaticLayoutSourceProbeCache, + resetNvidiaCudaAvailabilityCache, resolveExperimentalNvidiaCudaExportScriptPath, resolveNativeStaticLayoutFpsFields, resolveNvidiaCudaCursorAssets, resolveNvidiaCudaNativeFps, resolveNvidiaCudaNativeSummaryMetrics, resolveNvidiaCudaOverlaySidecarSummaryMetrics, + resolveNvidiaCudaStrictHevcHardFail, resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics, shouldCreateNativeStaticLayoutSourceProxy, validateNativeStaticLayoutSourceProxyMetadata, @@ -129,6 +144,13 @@ function resetFsAccessMock() { }); } +// The NVIDIA CUDA availability cache is session-scoped; reset it after every +// test so the mocked wrapper/GPU probes from one test never leak into another. +afterEach(() => { + resetNvidiaCudaAvailabilityCache(); + resetNativeStaticLayoutSourceProbeCache(); +}); + function createNvidiaCudaSkipOptions( overrides: Partial = {}, ): NativeStaticLayoutExportOptions { @@ -283,6 +305,137 @@ describe("native static-layout source proxy", () => { }); }); +describe("native static-layout source probe cache", () => { + const baseMetadata = { + width: 1920, + height: 1080, + duration: 45, + frameRate: 30, + codec: "h264 (High)", + hasAudio: true, + audioCodec: "aac", + }; + + function baseEntry() { + return { + identity: { + canonicalPath: "C:\\recordings\\session.mp4", + device: 1, + inode: 987654, + size: 5_000_000_000, + mtimeMs: 1_700_000_000_000, + ctimeMs: 1_700_000_000_000, + }, + requestedCodec: "hevc", + encodingMode: "quality", + encoderPreference: "hardware", + metadata: { ...baseMetadata }, + }; + } + + function baseCurrent() { + return { + canonicalPath: "C:\\recordings\\session.mp4", + device: 1, + inode: 987654, + size: 5_000_000_000, + mtimeMs: 1_700_000_000_000, + ctimeMs: 1_700_000_000_000, + requestedCodec: "hevc", + encodingMode: "quality", + encoderPreference: "hardware", + }; + } + + it("reuses a probe only on an exact identity and route match", () => { + expect(canReuseNativeStaticLayoutSourceProbe(baseEntry(), baseCurrent())).toBe(true); + }); + + it("never reuses when there is no cached entry", () => { + expect(canReuseNativeStaticLayoutSourceProbe(undefined, baseCurrent())).toBe(false); + }); + + it("invalidates on canonical path mismatch (never path-only reuse from another file)", () => { + const entry = baseEntry(); + entry.identity.canonicalPath = "C:\\recordings\\other.mp4"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on device mismatch", () => { + const entry = baseEntry(); + entry.identity.device = 2; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on inode mismatch", () => { + const entry = baseEntry(); + entry.identity.inode = 111; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on size change (mutation)", () => { + const entry = baseEntry(); + entry.identity.size = 5_000_000_001; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on mtime change (mutation)", () => { + const entry = baseEntry(); + entry.identity.mtimeMs = 1_700_000_100_000; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on ctime change (mutation)", () => { + const entry = baseEntry(); + entry.identity.ctimeMs = 1_700_000_100_000; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the requested output codec changes (changed settings)", () => { + const entry = baseEntry(); + entry.requestedCodec = "h264"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the encoding mode changes (changed settings)", () => { + const entry = baseEntry(); + entry.encodingMode = "balanced"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates when the encoder preference changes (changed settings)", () => { + const entry = baseEntry(); + entry.encoderPreference = "cpu"; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("never reuses a probe with an unknown source codec (re-probe or fail closed)", () => { + const entry = baseEntry(); + entry.metadata = { ...baseMetadata, codec: "unknown" }; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("never reuses a probe with an empty source codec", () => { + const entry = baseEntry(); + entry.metadata = { ...baseMetadata, codec: "" }; + expect(canReuseNativeStaticLayoutSourceProbe(entry, baseCurrent())).toBe(false); + }); + + it("invalidates on any current route mismatch even when identity matches", () => { + const entry = baseEntry(); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + encodingMode: "speed", + }), + ).toBe(false); + const current = baseCurrent(); + current.requestedCodec = "hevc"; + current.encoderPreference = "hardware"; + expect(canReuseNativeStaticLayoutSourceProbe(baseEntry(), current)).toBe(true); + }); +}); + describe("getNvidiaCudaAudioExportSkipReason", () => { it("allows video-only CUDA exports by default", () => { withNvidiaCudaAudioOverride(undefined, () => { @@ -599,6 +752,119 @@ describe("getExperimentalNvidiaCudaExportSkipReason", () => { }); }); +describe("NVIDIA CUDA availability cache", () => { + afterEach(() => { + resetNvidiaCudaAvailabilityCache(); + }); + + it("reuses the wrapper-path and GPU probes across capability and route queries", async () => { + await withPackagedCudaCandidate( + { gpuDevice: [{ vendorId: 0x10de, deviceString: "NVIDIA GeForce GTX 1650" }] }, + async () => { + const first = await getNativeExportCapabilities(); + const second = await getNativeExportCapabilities(); + const route = await getExperimentalNvidiaCudaExportSkipReason( + createNvidiaCudaSkipOptions({ experimentalNvidiaCudaExport: true }), + ); + + if (process.platform === "win32") { + // Two capabilities + a route decision share a single GPU probe. + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(1); + expect(route).toBeNull(); + } else { + expect(route).toBe("not-windows"); + } + expect(second.nvidiaCuda).toEqual(first.nvidiaCuda); + }, + ); + }); + + it("invalidates the cache when a relevant environment override changes the helper path", async () => { + const scriptEnv = "RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT"; + const exportEnv = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + const originalScript = process.env[scriptEnv]; + const originalExport = process.env[exportEnv]; + const originalIsPackaged = electronAppMock.isPackaged; + const customScript = "C:\\custom\\nvidia\\run-mp4-pipeline.mjs"; + electronAppMock.isPackaged = true; + electronAppMock.getGPUInfo.mockResolvedValue({ + gpuDevice: [{ vendorId: 0x10de, deviceString: "NVIDIA GeForce GTX 1650" }], + }); + delete process.env[scriptEnv]; + delete process.env[exportEnv]; + + try { + fsMocks.access.mockResolvedValue(undefined); + const first = await getNativeExportCapabilities(); + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(1); + + process.env[scriptEnv] = customScript; + fsMocks.access.mockImplementation(async (candidate: string) => { + if (candidate === customScript) { + return; + } + throw new Error(`missing ${candidate}`); + }); + + const second = await getNativeExportCapabilities(); + expect(electronAppMock.getGPUInfo.mock.calls.length).toBe(2); + if (process.platform === "win32") { + expect(second.nvidiaCuda.hasWrapper).toBe(true); + } else { + expect(second.nvidiaCuda).toBe(first.nvidiaCuda); + } + } finally { + if (originalScript === undefined) { + delete process.env[scriptEnv]; + } else { + process.env[scriptEnv] = originalScript; + } + if (originalExport === undefined) { + delete process.env[exportEnv]; + } else { + process.env[exportEnv] = originalExport; + } + electronAppMock.isPackaged = originalIsPackaged; + electronAppMock.getGPUInfo.mockReset(); + electronAppMock.getGPUInfo.mockResolvedValue({ gpuDevice: [] }); + resetFsAccessMock(); + resetNvidiaCudaAvailabilityCache(); + } + }); + + it("reports an unavailable GPU consistently for capability and route queries", async () => { + await withPackagedCudaCandidate( + { gpuDevice: [{ vendorId: 0x8086, deviceString: "Intel UHD Graphics" }] }, + async () => { + const capabilities = await getNativeExportCapabilities(); + const route = await getExperimentalNvidiaCudaExportSkipReason( + createNvidiaCudaSkipOptions({ experimentalNvidiaCudaExport: true }), + ); + if (process.platform === "win32") { + expect(capabilities.nvidiaCuda.available).toBe(false); + expect(capabilities.nvidiaCuda.skipReason).toBe("nvidia-gpu-unavailable"); + expect(capabilities.nvidiaCuda.hasNvidiaGpu).toBe(false); + expect(route).toBe("nvidia-gpu-unavailable"); + } else { + expect(capabilities.nvidiaCuda.skipReason).toBe("not-windows"); + expect(route).toBe("not-windows"); + } + }, + ); + }); + + it("keeps strict HEVC Hardware CUDA-only hard-fail behavior unchanged", () => { + expect(resolveNvidiaCudaStrictHevcHardFail(true, false, "nvidia-gpu-unavailable")).toBe( + "HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (nvidia-gpu-unavailable) (noCpuFallback:true)", + ); + expect(resolveNvidiaCudaStrictHevcHardFail(true, false, null)).toBe( + "HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (cursor-atlas-unavailable) (noCpuFallback:true)", + ); + expect(resolveNvidiaCudaStrictHevcHardFail(true, true, "env-disabled")).toBeNull(); + expect(resolveNvidiaCudaStrictHevcHardFail(false, false, "env-disabled")).toBeNull(); + }); +}); + describe("resolveExperimentalNvidiaCudaExportScriptPath", () => { it("prefers the packaged app.asar.unpacked CUDA wrapper over the virtual app.asar copy", async () => { const envName = "RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT"; @@ -787,6 +1053,41 @@ describe("buildNativeStaticLayoutOverlayManifest", () => { }); expect("effectiveFrameCount" in manifest.layers[0]).toBe(false); }); + + it("serializes a cursor-sprite layer with its additive positions sidecar fields", () => { + const manifest = buildNativeStaticLayoutOverlayManifest([ + { + id: "cursor-sprite", + order: 1, + kind: "cursor-sprite", + path: "cursor.sprite", + positionsPath: "cursor.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameRate: 30, + durationSec: 2, + frameCount: 60, + positions: Array.from({ length: 60 }, () => ({ x: 0, y: 0 })), + pixelFormat: "rgba", + }, + ]); + + expect(manifest.layers[0]).toEqual({ + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + path: "cursor.sprite", + positionsPath: "cursor.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameCount: 60, + }); + expect("effectiveFrameCount" in manifest.layers[0]).toBe(false); + }); }); describe("getNativeStaticLayoutOverlayExpectedSidecarBytes", () => { @@ -1129,6 +1430,22 @@ describe("buildExperimentalNvidiaCudaStaticLayoutArgs", () => { expect(args).not.toContain("--temporal-blur-sample-count"); }); + it("rejects temporal blur plans below the minimum sample count instead of silently dropping the effect", () => { + expect(() => + buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + temporalBlur: { + sampleCount: 2, + shutterFraction: 0.62, + weightCurvePower: 1.5, + }, + }), + "output.mp4", + "work", + ), + ).toThrow(/unsupported-temporal-motion-blur/); + }); + it("keeps the D3D11 builder free of overlay manifest args", () => { const args = buildExperimentalWindowsGpuStaticLayoutArgs( createNvidiaCudaSkipOptions({ @@ -1344,6 +1661,58 @@ describe("muxExportedVideoAudioBuffer", () => { expect(result.outputPath).toMatch(/recordly-export-video-/); }); + + it("swallows the child-process error from a terminating kill of an already-exited child", async () => { + const spawnMock = vi.mocked(spawn); + spawnMock.mockClear(); + const child = new EventEmitter() as unknown as { + stdout: { on: (event: string, cb: (chunk: Buffer) => void) => void }; + stderr: { on: (event: string, cb: (chunk: Buffer) => void) => void }; + pid: number; + killed: boolean; + kill: (signal?: string) => boolean; + } & ReturnType; + (child as unknown as { stdout: unknown }).stdout = { on: vi.fn() }; + (child as unknown as { stderr: unknown }).stderr = { on: vi.fn() }; + (child as unknown as { pid: number }).pid = 4242; + let killedWith: string[] = []; + (child as unknown as { killed: boolean }).killed = false; + (child as unknown as { kill: unknown }).kill = (signal?: string) => { + if (signal) { + killedWith.push(signal); + } + // Killing an already-exited child on Windows emits an 'error' event + // asynchronously. With the immediate no-op listener attached (Module 3 + // fix) this is handled, not an uncaught exception; the dedicated + // once-handler (attached right after) then settles the promise. + queueMicrotask(() => { + child.emit("error", new Error('The process "4242" not found.')); + }); + return false; + }; + spawnMock.mockReturnValue(child); + + const session = { + terminating: true, + currentProcess: null as unknown, + }; + const error = await muxNativeVideoExportAudio( + "video.mp4", + { + audioMode: "copy-source", + audioSourceCodec: "aac", + audioSourcePath: "source-audio.mp4", + } as never, + undefined, + session as never, + ).catch((caught: unknown) => caught); + + // The child was terminated (SIGKILL) and its error event was handled + // rather than surfacing as an uncaught "process not found" noise line. + expect(killedWith).toContain("SIGKILL"); + expect(error).toBeInstanceOf(Error); + spawnMock.mockReset(); + }); }); describe("buildNativeVideoAudioMuxArgs", () => { @@ -1699,6 +2068,20 @@ describe("resolveNvidiaCudaNativeSummaryMetrics", () => { it("returns an empty object for missing native summaries", () => { expect(resolveNvidiaCudaNativeSummaryMetrics(undefined)).toEqual({}); }); + + it("surfaces the separated overlay host-read and H2D enqueue stage fields", () => { + expect( + resolveNvidiaCudaNativeSummaryMetrics({ + overlayHostReadMs: 6.2, + overlayH2DEnqueueMs: 9.4, + overlayUploadMs: 18.1, + }), + ).toEqual({ + overlayHostReadMs: 6.2, + overlayH2DEnqueueMs: 9.4, + overlayUploadMs: 18.1, + }); + }); }); describe("resolveNvidiaCudaNativeFps", () => { @@ -1716,6 +2099,16 @@ describe("resolveNvidiaCudaNativeFps", () => { expect(resolveNvidiaCudaNativeFps({ nativeSummary: { fps: 0 } })).toBeUndefined(); expect(resolveNvidiaCudaNativeFps(undefined)).toBeUndefined(); }); + + it("never falls back to the configured output fps and labels it measured speed", () => { + // summary.fps is the configured stream FPS (30), not a measured encode + // rate. When the helper reports no measured FPS the resolver must return + // undefined rather than surfacing 30 as nativeFps. + expect(resolveNvidiaCudaNativeFps({ fps: 30, nativeSummary: {} })).toBeUndefined(); + expect( + resolveNvidiaCudaNativeFps({ fps: 30, nativeSummary: { measuredFps: 0 } }), + ).toBeUndefined(); + }); }); describe("validateNvidiaCudaStageMetricInvariants", () => { @@ -2037,6 +2430,41 @@ describe("parseWindowsGpuExportProgressLine", () => { fpsSource: "native", }); }); + + it("never emits a preparation-inclusive estimate for finalizing progress", () => { + // During finalizing the helper has already reported measured encode + // FPS; a frames/since-spawn estimate at this stage would include source + // preparation and misrepresent the display as measured speed. + expect( + resolveNativeStaticLayoutFpsFields( + { currentFrame: 300, totalFrames: 300, percentage: 100, stage: "finalizing" }, + 3_900, + ), + ).toEqual({ + averageFps: undefined, + estimatedFps: undefined, + fpsSource: undefined, + }); + }); + + it("still reports the helper measured encode FPS on finalizing progress", () => { + expect( + resolveNativeStaticLayoutFpsFields( + { + currentFrame: 300, + totalFrames: 300, + percentage: 100, + stage: "finalizing", + averageFps: 135.4, + }, + 3_900, + ), + ).toEqual({ + averageFps: 135.4, + estimatedFps: undefined, + fpsSource: "native", + }); + }); }); describe("formatNativeStaticLayoutZoomTelemetryLines", () => { it("writes renderer zoom-blur columns for the CUDA compositor", () => { @@ -2118,6 +2546,63 @@ describe("mapNvidiaCudaWrapperProgressPercentage", () => { }); }); +describe("buildNvidiaCudaPrepareProgress", () => { + it.each([ + ["encoder-probe"] as const, + ["source-validation"] as const, + ["wrapper-launch"] as const, + ["cuda-nvenc-init"] as const, + ["first-frame"] as const, + ])("labels every preparation substate (%s) as NVIDIA CUDA compositor", (substate) => { + const progress = buildNvidiaCudaPrepareProgress("sess-1", substate, 600, 1_234); + expect(progress).toMatchObject({ + sessionId: "sess-1", + stage: "preparing", + substate, + backend: "nvidia-cuda-compositor", + currentFrame: 0, + totalFrames: 600, + elapsedMs: 1_234, + }); + // Preparation substates must never fabricate FPS or claim encode speed. + expect(progress.averageFps).toBeUndefined(); + expect(progress.instantFps).toBeUndefined(); + expect(progress.estimatedFps).toBeUndefined(); + expect(progress.fpsSource).toBeUndefined(); + }); + + it("stays within the preparing window and never claims a rendered frame", () => { + const orderedSubstates = [ + "encoder-probe", + "source-validation", + "wrapper-launch", + "cuda-nvenc-init", + "first-frame", + ] as const; + const percentages = orderedSubstates.map( + (substate) => buildNvidiaCudaPrepareProgress(undefined, substate, 600, 100).percentage, + ); + // Additive progression, all display-only within the preparing window. + for (let index = 0; index < percentages.length; index += 1) { + expect(percentages[index]).toBeGreaterThan(0); + expect(percentages[index]).toBeLessThanOrEqual(3); + if (index > 0) { + expect(percentages[index]).toBeGreaterThan(percentages[index - 1]); + } + } + expect(buildNvidiaCudaPrepareProgress(undefined, "encoder-probe", 600, 100)).toHaveProperty( + "currentFrame", + 0, + ); + }); + + it("clamps the total frames to a positive integer", () => { + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 0, 10).totalFrames).toBe(1); + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 2.9, 10).totalFrames).toBe(2); + expect(buildNvidiaCudaPrepareProgress("s", "first-frame", 600, -5).elapsedMs).toBe(0); + }); +}); + describe("hasNativeStaticLayoutProgressAdvanced", () => { it("treats repeated preparation heartbeats as stalled until real progress arrives", () => { const previous = { currentFrame: 0, percentage: 2.5 }; @@ -2331,6 +2816,130 @@ describe("native cursor atlas ownership", () => { }); }); +describe("native webcam ownership", () => { + function createWebcamOverlayLayer(): NativeStaticLayoutOverlayLayer { + return { + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + path: "cursor-sprite.rgba", + positionsPath: "cursor-sprite.positions.json", + x: 0, + y: 0, + width: 32, + height: 32, + frameRate: 30, + durationSec: 10, + frameCount: 300, + pixelFormat: "rgba", + positions: Array.from({ length: 300 }, () => ({ x: 1, y: 1 })), + }; + } + + it("passes webcam args and the overlay manifest through when the CUDA compositor owns the webcam", () => { + const args = buildExperimentalNvidiaCudaStaticLayoutArgs( + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamLeft: 32, + webcamTop: 48, + webcamSize: 240, + webcamRadius: 18, + webcamMirror: false, + webcamNativeOwned: true, + overlayManifestPath: "overlay-manifest.json", + overlayLayers: [createWebcamOverlayLayer()], + }), + "output.mp4", + "work", + ); + + // The sidecar excluded webcam pixels, so the CUDA wrapper must receive + // BOTH the webcam input and the overlay manifest (cursor sprite); + // dropping either would lose the webcam or cursor and baking the webcam + // again would double-render. + expect(args).toEqual(expect.arrayContaining(["--webcam-input", "webcam.mp4"])); + expect(args).toEqual(expect.arrayContaining(["--webcam-x", "32"])); + expect(args).toEqual(expect.arrayContaining(["--webcam-size", "240"])); + expect(args).toEqual( + expect.arrayContaining(["--overlay-manifest", "overlay-manifest.json"]), + ); + }); + + it("refuses a native-owned webcam when the CUDA route cannot run", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamSize: 240, + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: false, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor/i, + ); + }); + + it("refuses a native-owned webcam without an input path", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Native webcam ownership requires a webcam input path/i, + ); + }); + + it("refuses a native-owned webcam without a positive size", async () => { + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).toMatch( + /Native webcam ownership requires a positive webcam size/i, + ); + }); + + it("allows a native-owned webcam when the CUDA route is explicitly opted in on Windows", async () => { + if (process.platform !== "win32") { + return; + } + + const error = await exportNativeStaticLayoutVideo( + "ffmpeg", + createNvidiaCudaSkipOptions({ + webcamInputPath: "webcam.mp4", + webcamSize: 240, + webcamNativeOwned: true, + experimentalWindowsGpuCompositor: true, + experimentalNvidiaCudaExport: true, + }), + ).catch((caught: unknown) => caught); + + // The preflight guard must not reject the CUDA-opt-in route; any later + // failure is a runtime/skip error, not a webcam-ownership refusal. + expect(error).toBeInstanceOf(Error); + expect((error as Error).message).not.toMatch( + /Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor/i, + ); + }); +}); + describe("resolveNvidiaCudaOverlaySidecarSummaryMetrics", () => { it("surfaces dimensions and physical/effective frame counts for deduped sidecars", () => { expect( @@ -2470,19 +3079,74 @@ describe("native summary metric invariants (module 3)", () => { ).toEqual(["temporalBlurStationaryFrames 41 exceeds temporalBlurFrames 40"]); }); - it("rejects temporal cache counters that exceed the precomposed frame budget", () => { + it("does not count cache builds against the precomposed frame budget", () => { + // Mirrors the reported false positive: builds count cache allocations/ + // segments, hits and precomposedFrames count per-frame reuse. builds must + // NOT be added to the frame budget. + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 4, + temporalBgCacheBuilds: 1, + temporalBgCacheHits: 4, + }, + }), + ).toEqual([]); + }); + + it("rejects temporal cache hits that exceed the precomposed frame budget", () => { expect( validateNvidiaCudaStageMetricInvariants({ nativeSummary: { temporalBlurFrames: 40, temporalBlurBgPrecomposedFrames: 30, temporalBgCacheBuilds: 20, - temporalBgCacheHits: 15, + temporalBgCacheHits: 31, }, }), - ).toEqual([ - "temporalBgCacheBuilds 20 + temporalBgCacheHits 15 exceed temporalBlurBgPrecomposedFrames 30", - ]); + ).toEqual(["temporalBgCacheHits 31 exceeds temporalBlurBgPrecomposedFrames 30"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBlurBgPrecomposedFrames: 30, + temporalBgCacheBuilds: 40, + temporalBgCacheHits: 30, + }, + }), + ).toEqual([]); + }); + + it("binds temporal cache hits to the temporal frame budget when precomposed is absent", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBlurFrames: 40, + temporalBgCacheHits: 41, + temporalBgCacheBuilds: 10, + }, + }), + ).toEqual(["temporalBgCacheHits 41 exceeds temporalBlurFrames 40"]); + }); + + it("rejects negative temporal cache counters", () => { + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBgCacheBuilds: -1, + temporalBgCacheHits: 4, + }, + }), + ).toEqual(["temporalBgCacheBuilds -1 must be non-negative"]); + expect( + validateNvidiaCudaStageMetricInvariants({ + nativeSummary: { + temporalBgCacheBuilds: 1, + temporalBgCacheHits: -4, + }, + }), + ).toEqual(["temporalBgCacheHits -4 must be non-negative"]); }); it("rejects overlay static-region blends beyond overlay blend frames", () => { @@ -2890,3 +3554,468 @@ describe("tiled overlay integration (native-video module)", () => { ).toBeNull(); }); }); + +describe("prewarmNativeExportCaches", () => { + const nvidiaIdentityStat = { + dev: 123, + ino: 456, + size: 1_000_000, + mtimeMs: 1_000, + ctimeMs: 2_000, + }; + const cacheableMetadataProbe = + "Duration: 00:00:02.00, start: 0.000000\n Stream #0:0: Video: h264 (High), yuv420p, 1920x1080, 30 fps"; + + function countMetadataProbeCalls(): number { + return execFileMock.mock.calls.filter( + (call) => Array.isArray(call[1]) && call[1].includes("-i"), + ).length; + } + + function restoreExecFileMock(): void { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null); + return { stdout: "", stderr: "" } as unknown; + }) as never); + } + + beforeEach(() => { + resetNativeStaticLayoutSourceProbeCache(); + resetNvidiaCudaAvailabilityCache(); + fsMocks.realpath.mockResolvedValue("canonical.mp4"); + fsMocks.stat.mockResolvedValue(nvidiaIdentityStat as never); + execFileMock.mockClear(); + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null); + return { stdout: "", stderr: "" } as unknown; + }) as never); + }); + + afterEach(() => { + restoreExecFileMock(); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + }); + + it("warms source metadata and reuses exact cache keys (codec + encoder preference)", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + + const first = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(countMetadataProbeCalls()).toBe(1); + expect(parseNativeVideoMetadataProbeOutput(cacheableMetadataProbe)).not.toBeNull(); + expect( + parseNativeVideoMetadataProbeOutput("\n" + cacheableMetadataProbe)?.codec, + "probe-result-codec", + ).toBe("h264 (High)"); + expect(first.sourceMetadataCached, `skip=${JSON.stringify(first.skipReasons)}`).toBe(true); + + // Exact same route reuses the cache: no second probe. + const second = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(second.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A different encoder preference is a distinct exact key: it re-probes. + const changedPref = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "hardware", + encodingMode: "balanced", + }); + expect(changedPref.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(2); + }); + + it("treats encoding mode as part of the exact cache key (prewarm/export alignment)", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + + // Warm with the persisted/export default (balanced) once. + const first = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(first.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A real export resolving the same route with balanced hits the cache. + const hit = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(hit.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(1); + + // A different encoding mode is a distinct exact key and re-probes. + const otherMode = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "quality", + }); + expect(otherMode.sourceMetadataCached).toBe(true); + expect(countMetadataProbeCalls()).toBe(2); + }); + + it("reports source metadata as uncached when the probe fails without throwing", async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + try { + cb(new Error("probe failed") as never); + } catch { + // swallow + } + return { stdout: "", stderr: "" } as unknown; + }) as never); + + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(outcome.sourceMetadataCached).toBe(false); + expect(outcome.skipReasons).toContain("source-metadata-unavailable"); + expect(outcome.resolvedEncoders).toContain("h264_nvenc"); + }); + + it("reports CUDA availability when the runtime probe succeeds", async () => { + await withPackagedCudaCandidate( + { + gpuDevice: [{ vendorId: 0x10de, deviceString: "RTX 4090" }], + }, + async () => { + execFileMock.mockImplementation((( + _cmd: string, + _args: string[], + _opts: unknown, + cb: (err: Error | null) => void, + ) => { + cb(null as never, { stdout: cacheableMetadataProbe, stderr: "" }); + return { stdout: cacheableMetadataProbe, stderr: "" } as unknown; + }) as never); + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "h264", + encoderPreference: "auto", + encodingMode: "balanced", + }); + expect(outcome.cudaAvailabilityResolved).toBe(true); + }, + ); + }); + + it("does not fabricate CUDA readiness from an unavailable runtime probe", async () => { + const outcome = await prewarmNativeExportCaches({ + inputPath: "input.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + encodingMode: "balanced", + }); + expect(outcome.cudaAvailabilityResolved).toBe(false); + expect(outcome.skipReasons.some((reason) => reason.startsWith("cuda-unavailable"))).toBe( + true, + ); + }); +}); + +describe("capability-only prewarm cancellation (native-video)", () => { + function fakeCapabilityChild() { + const child = new EventEmitter() as { + stdout: unknown; + stderr: unknown; + killedWith: string[]; + kill: (signal?: string) => void; + }; + child.stdout = {}; + child.stderr = {}; + child.killedWith = []; + child.kill = (signal?: string) => { + if (signal) { + child.killedWith.push(signal); + } + }; + return child; + } + + it("cancels every in-flight capability-only prewarm child before a real NVENC session opens", () => { + const childA = fakeCapabilityChild(); + const childB = fakeCapabilityChild(); + const unregisterA = registerCapabilityOnlyPrewarmChild(childA); + const unregisterB = registerCapabilityOnlyPrewarmChild(childB); + + const cancelled = cancelInFlightCapabilityOnlyPrewarms(); + + expect(cancelled).toBe(2); + expect(childA.killedWith).toContain("SIGKILL"); + expect(childB.killedWith).toContain("SIGKILL"); + + // Unregister after settle is a no-op (set already cleared). + unregisterA(); + unregisterB(); + }); + + it("is harmless when no capability-only prewarm child is running", () => { + expect(cancelInFlightCapabilityOnlyPrewarms()).toBe(0); + }); + + it("does not cancel a child that already finished (unregistered)", () => { + const child = fakeCapabilityChild(); + const unregister = registerCapabilityOnlyPrewarmChild(child); + unregister(); + expect(cancelInFlightCapabilityOnlyPrewarms()).toBe(0); + expect(child.killedWith).toHaveLength(0); + }); +}); + +describe("native static-layout lazy FFmpeg encoder resolution", () => { + const METADATA_H264 = + "Duration: 00:00:02.00, start: 0.000000\n Stream #0:0: Video: h264 (High), yuv420p, 1920x1080, 30 fps"; + const FFPROBE_STATS = JSON.stringify({ + streams: [ + { + duration: "1.9999", + nb_read_frames: "10", + avg_frame_rate: "5/1", + r_frame_rate: "5/1", + }, + ], + }); + const CUDA_SUMMARY = JSON.stringify({ + success: true, + outputCodec: "h264", + targetFrames: 10, + durationSec: 2, + nativeSummary: { success: true, frames: 10 }, + outputVideo: { duration: "1.999900", nb_frames: "10" }, + }); + + function makeOptions( + overrides: Partial = {}, + ): NativeStaticLayoutExportOptions { + return { + inputPath: "input.mp4", + width: 1920, + height: 1080, + frameRate: 5, + bitrate: 8_000_000, + encodingMode: "balanced", + durationSec: 2, + contentWidth: 1920, + contentHeight: 1080, + offsetX: 0, + offsetY: 0, + backgroundColor: "#101010", + audioOptions: { audioMode: "none" }, + ...overrides, + }; + } + + function routeExecFileTo(encodersStdout: string) { + execFileMock.mockImplementation((( + _cmd: string, + args: string[], + _opts: unknown, + cb: (err: Error | null, res?: { stdout: string; stderr: string }) => void, + ) => { + if (Array.isArray(args) && args.includes("-encoders")) { + cb(null, { stdout: encodersStdout, stderr: "" }); + return { stdout: encodersStdout, stderr: "" } as unknown; + } + if (Array.isArray(args) && args.includes("-select_streams")) { + cb(null, { stdout: FFPROBE_STATS, stderr: "" }); + return { stdout: FFPROBE_STATS, stderr: "" } as unknown; + } + // FFmpeg metadata probe (-i source) -> valid H.264 source (no proxy). + cb(null, { stdout: "", stderr: METADATA_H264 }); + return { stdout: "", stderr: METADATA_H264 } as unknown; + }) as never); + } + + function fakeSpawnChild() { + const stdout = new EventEmitter(); + const stderr = new EventEmitter(); + const child = new EventEmitter() as EventEmitter & { + stdout: EventEmitter; + stderr: EventEmitter; + stdin: { end: () => void }; + kill: (signal?: string) => boolean; + }; + child.stdout = stdout; + child.stderr = stderr; + child.stdin = { end: () => undefined }; + child.kill = () => true; + return child; + } + + function countEncodersProbes(): number { + return execFileMock.mock.calls.filter( + (call) => Array.isArray(call[1]) && call[1].includes("-encoders"), + ).length; + } + + beforeEach(() => { + resetNativeStaticLayoutSourceProbeCache(); + resetNvidiaCudaAvailabilityCache(); + vi.mocked(spawn).mockClear(); + fsMocks.access.mockResolvedValue(undefined); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 } as never); + }); + + afterEach(() => { + fsMocks.access.mockImplementation(async () => { + throw new Error("missing"); + }); + fsMocks.stat.mockResolvedValue({ size: 5_000_000_000 }); + delete process.env.RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT; + delete process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT; + }); + + it("does not invoke the FFmpeg encoder probe when the NVIDIA CUDA route is selected", async () => { + routeExecFileTo(""); + const cudaEnv = "RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT"; + process.env[cudaEnv] = "1"; + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT = "run-mp4-pipeline.mjs"; + + const spawnMock = vi.mocked(spawn); + const child = fakeSpawnChild(); + spawnMock.mockImplementation((cmd: string) => { + // Only the CUDA compositor wrapper (node -- run-mp4-pipeline.mjs) may be + // spawned; no ffmpeg encoder probe may happen before/on the CUDA route. + expect(cmd).toBe(process.execPath); + return child; + }); + + const pending = exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ experimentalWindowsGpuCompositor: true }), + ); + await vi.waitFor(() => { + expect(spawnMock).toHaveBeenCalled(); + }); + // CUDA path reached the wrapper spawn without every probing the encoder. + expect(countEncodersProbes()).toBe(0); + child.stdout.emit("data", Buffer.from(CUDA_SUMMARY)); + child.emit("close", 0, null); + + const result = await pending; + expect(result.route).toBe("nvidia-cuda-compositor"); + expect(result.encoderName).toBe("nvidia-cuda-compositor"); + // The CUDA route succeeded with zero FFmpeg encoder listing probes. + expect(countEncodersProbes()).toBe(0); + }); + + it("still probes/resolves the FFmpeg encoder on the raw/FFmpeg fallback branch", async () => { + // Only libx264 is available, so exactly one probe spawn happens for it. + routeExecFileTo(" V....D libx264"); + const spawnMock = vi.mocked(spawn); + spawnMock.mockImplementation((_cmd: string, args: string[]) => { + const probeChild = fakeSpawnChild(); + if (Array.isArray(args) && args.includes("pipe:0")) { + // Encoder probe succeeds -> resolveNativeVideoEncoder returns libx264. + queueMicrotask(() => probeChild.emit("close", 0, null)); + } else { + // FFmpeg render spawns fail, so the export rejects after resolution. + queueMicrotask(() => probeChild.emit("close", 1, null)); + } + return probeChild; + }); + + await expect( + exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ experimentalWindowsGpuCompositor: false }), + ), + ).rejects.toThrow(); + + // The FFmpeg/raw fallback probed and resolved the encoder (-encoders), + // unlike the CUDA route above. + expect(countEncodersProbes()).toBeGreaterThan(0); + }); + + it("strict HEVC Hardware hard-fails on CUDA runtime failure without webcam/zoom/timeline (no FFmpeg fallback/probe)", async () => { + // Strict HEVC Hardware: the generalized NVIDIA CUDA compositor is the ONLY + // acceptable route. A CUDA runtime failure with no webcam, zoom telemetry, + // or native timeline must NOT be swallowed by the outer GPU-block catch and + // turned into a full FFmpeg hevc_nvenc fallback; the original actionable + // CUDA/noCpuFallback error must reach the caller and no FFmpeg encoder probe + // or fallback spawn may happen. + routeExecFileTo(""); + process.env.RECORDLY_EXPERIMENTAL_NVIDIA_CUDA_EXPORT = "1"; + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT = "run-mp4-pipeline.mjs"; + const probesBefore = countEncodersProbes(); + + const spawnMock = vi.mocked(spawn); + const child = fakeSpawnChild(); + spawnMock.mockImplementation((cmd: string) => { + // Only the CUDA compositor wrapper may spawn; no Windows-GPU or FFmpeg + // fallback child may ever be launched on the strict route. + expect(cmd).toBe(process.execPath); + return child; + }); + + const pending = exportNativeStaticLayoutVideo( + "ffmpeg", + makeOptions({ + experimentalWindowsGpuCompositor: true, + videoCodec: "hevc", + encoderPreference: "hardware", + }), + ); + await vi.waitFor(() => { + expect(spawnMock).toHaveBeenCalled(); + }); + // CUDA runtime failure: the helper exits nonzero without a success summary. + child.emit("close", 1, null); + + await expect(pending).rejects.toThrow(/noCpuFallback:true/); + // The strict route reached only the single CUDA wrapper spawn; no FFmpeg + // encoder probe (-encoders) and no FFmpeg/GPU fallback spawn occurred. + expect(spawnMock).toHaveBeenCalledTimes(1); + // No FFmpeg encoder probe (-encoders) may have occurred for this export. + // countEncodersProbes() accumulates across the describe, so compare against + // the count captured before this export ran. + expect(countEncodersProbes()).toBe(probesBefore); + }); +}); diff --git a/electron/ipc/export/native-video.ts b/electron/ipc/export/native-video.ts index d834a8a00..de96e52fe 100644 --- a/electron/ipc/export/native-video.ts +++ b/electron/ipc/export/native-video.ts @@ -9,21 +9,26 @@ import { promisify } from "node:util"; import type { MessagePortMain, WebContents } from "electron"; import { app, powerSaveBlocker } from "electron"; import type { + NativeCursorSpriteOverlayLayer, NativeStaticLayoutOverlayLayer, NativeTiledOverlayLayerDescriptor, NativeTiledOverlayStorageDescriptor, } from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; import { getNativeStaticLayoutOverlayFrameByteSize, + NATIVE_CURSOR_SPRITE_LAYER_KIND, NATIVE_TILED_OVERLAY_STORAGE_VERSION, resolveNativeTiledOverlayMetrics, resolveNativeTiledOverlayRawFallbackReason, sortNativeStaticLayoutOverlayLayers, sortNativeTiledOverlayLayers, + validateNativeCursorSpriteOverlayLayer, validateNativeStaticLayoutOverlayLayer, validateNativeTiledOverlayStorageDescriptor, } from "../../../src/lib/exporter/nativeStaticLayoutOverlays"; +import { TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT } from "../../../src/lib/exporter/temporalMotionBlur"; import { getFfmpegBinaryPath, getFfprobeBinaryPath } from "../ffmpeg/binary"; +import { formatLogTs } from "../log"; import type { ExportEncoderPreference, ExportVideoCodec, @@ -150,11 +155,69 @@ export type NativeVideoExportSession = { framePortReady: boolean; nextFrameSequence: number; pendingFrameRequests: Map; - completedFrameRequestIds: Set; + /** + * Monotonic watermark of the highest accepted frame request id. Request ids + * are allocated strictly monotonically by the renderer + * (nextNativeVideoExportWriteRequestId++), and the strict sequence check + * below already rejects replays, so a bounded O(1) watermark replaces the + * per-session completed-id Set (which grew to ~1 entry per exported frame + * for the whole session). Reset together with nextFrameSequence whenever a + * new frame port is attached. + */ + highestAcceptedFrameRequestId: number; }; export const nativeVideoExportSessions = new Map(); +// In-flight NVIDIA CUDA/NVENC capability-only prewarm children, tracked here +// (not in native-prewarm) so a real native export can cancel them without the +// coordinator importing back into this module (avoiding a circular dependency). +// Each child opens a brief NVENC capability probe; cancelling them before a +// real export opens its own NVENC session prevents concurrent NVENC session +// contention on the GPU. Fire-and-forget: the coordinator never awaits these. +const activeCapabilityOnlyPrewarmChildren = new Set>(); + +/** + * Registers an in-flight capability-only prewarm child process. Returns a + * cleanup that removes it from the tracked set when it settles. Diagnostic/data + * only; never awaited by the coordinator (fire-and-forget). + */ +export function registerCapabilityOnlyPrewarmChild(child: ReturnType): () => void { + activeCapabilityOnlyPrewarmChildren.add(child); + return () => { + activeCapabilityOnlyPrewarmChildren.delete(child); + }; +} + +/** + * Terminates every in-flight capability-only prewarm child. Called when a real + * native export opens its NVENC session so the brief capability probe never + * contends with the real encode. Harmless if none are running. Returns how many + * children were killed (diagnostic only; never used for control flow). + */ +export function cancelInFlightCapabilityOnlyPrewarms(): number { + let cancelled = 0; + for (const child of activeCapabilityOnlyPrewarmChildren) { + try { + child.kill("SIGKILL"); + cancelled += 1; + } catch { + /* process may already be exited */ + } + } + activeCapabilityOnlyPrewarmChildren.clear(); + if (cancelled > 0) { + console.info( + formatLogTs(), + "[native-export] Cancelled in-flight CUDA capability-only prewarm children", + { + cancelled, + }, + ); + } + return cancelled; +} + function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null; } @@ -354,7 +417,7 @@ function handleNativeVideoExportFramePortMessage( return; } if ( - session.completedFrameRequestIds.has(requestId) || + requestId <= session.highestAcceptedFrameRequestId || session.pendingFrameRequests.has(requestId) ) { sendNativeVideoExportFramePortError( @@ -409,7 +472,7 @@ function handleNativeVideoExportFramePortMessage( } session.pendingFrameRequests.set(requestId, { sequence }); - session.completedFrameRequestIds.add(requestId); + session.highestAcceptedFrameRequestId = requestId; session.nextFrameSequence += 1; void enqueueNativeVideoExportFrameWrite(session, value.frame) .then(() => { @@ -484,7 +547,7 @@ export function attachNativeVideoExportFramePort( session.framePortReady = false; session.nextFrameSequence = 0; session.pendingFrameRequests.clear(); - session.completedFrameRequestIds.clear(); + session.highestAcceptedFrameRequestId = -1; port.on("message", (event) => { if (session.framePort !== port) { return; @@ -531,6 +594,18 @@ export interface NativeStaticLayoutTimelineSegment { speed: number; } +/** Fixed-position rgba raw overlay layer or packed cursor-sprite layer. */ +export type NativeStaticLayoutOverlayLayerUnion = + | NativeStaticLayoutOverlayLayer + | NativeCursorSpriteOverlayLayer; + +/** Discriminates a cursor-sprite layer from a fixed-position rgba layer. */ +export function isCursorSpriteOverlayLayer( + layer: NativeStaticLayoutOverlayLayerUnion, +): layer is NativeCursorSpriteOverlayLayer { + return (layer as { kind?: string }).kind === NATIVE_CURSOR_SPRITE_LAYER_KIND; +} + export interface NativeStaticLayoutExportOptions { sessionId?: string; inputPath: string; @@ -543,7 +618,7 @@ export interface NativeStaticLayoutExportOptions { bitrate: number; encodingMode: NativeExportEncodingMode; durationSec: number; - overlayLayers?: NativeStaticLayoutOverlayLayer[]; + overlayLayers?: NativeStaticLayoutOverlayLayerUnion[]; /** * Optional tiled/delta overlay layers (sparse overlay optimization). The * renderer emits this instead of raw overlayLayers for sparse content; the @@ -572,6 +647,18 @@ export interface NativeStaticLayoutExportOptions { webcamShadowIntensity?: number; webcamMirror?: boolean; webcamTimeOffsetMs?: number; + /** + * True when the renderer excluded webcam pixels from the overlay sidecars + * and the generalized NVIDIA CUDA compositor owns the webcam overlay natively + * (--webcam-input contract). The CUDA compositor must draw the webcam on top + * of the composed video; stripping the webcam args while the sidecar already + * baked them would double-render, and passing them while the sidecar still + * contained the webcam would double-render too. Absent/false keeps the + * baked-webcam contract. Only the strict HEVC Hardware CUDA route sets this + * (that route hard-fails instead of falling back), so a fallback that cannot + * draw a native webcam can never silently drop it. + */ + webcamNativeOwned?: boolean; cursorTelemetry?: Array<{ timeMs: number; cx: number; @@ -632,10 +719,26 @@ export interface NativeStaticLayoutExportOptions { nvidiaCudaForceVideoOnly?: boolean; } +/** + * Additive CUDA-preparation sub-stage. Emitted as display-only progress during + * the native NVIDIA CUDA compositor startup so the renderer can show what the + * main/native side is doing (encoder probe -> source validation -> wrapper + * launch -> CUDA/NVENC initialization -> first-frame readiness). These are + * preparation substates, never encode throughput: they must not fabricate FPS. + */ +export type NativeStaticLayoutPrepareSubstate = + | "encoder-probe" + | "source-validation" + | "wrapper-launch" + | "cuda-nvenc-init" + | "first-frame"; + export interface NativeStaticLayoutExportProgress { sessionId?: string; backend?: NativeStaticLayoutBackend; stage?: "preparing" | "finalizing"; + /** Additive preparation sub-stage (see NativeStaticLayoutPrepareSubstate). */ + substate?: NativeStaticLayoutPrepareSubstate; elapsedMs?: number; averageFps?: number; instantFps?: number; @@ -786,9 +889,9 @@ export interface NvidiaCudaNativeSummary { /** Temporal blur invariant-background cache hits (additive). */ temporalBgCacheHits?: number; /** - * Overlay host-read wall time in ms. The helper currently reports only the - * combined overlayUploadMs; these fields stay absent until main.cu separates - * host reads from H2D enqueues. + * Overlay host-read wall time in ms. main.cu reports the separated + * host-read and H2D enqueue spans; these fields are emitted when the helper + * measured them (additive over the encode run). */ overlayHostReadMs?: number; overlayH2DEnqueueMs?: number; @@ -1071,17 +1174,17 @@ export function resolveNvidiaCudaOverlaySidecarSummaryMetrics( return {}; } + // A cursor-sprite layer is a packed strip (no effectiveFrameCount dedup); its + // physical frame count equals the logical output duration. + const effectiveFrameCount = (layer as { effectiveFrameCount?: number }).effectiveFrameCount; const metrics: Partial = { overlayWidth: Math.max(1, Math.round(layer.width)), overlayHeight: Math.max(1, Math.round(layer.height)), overlayFrameCount: Math.max(1, Math.round(layer.frameCount)), - overlayPhysicalFrames: Math.max( - 1, - Math.round(layer.effectiveFrameCount ?? layer.frameCount), - ), + overlayPhysicalFrames: Math.max(1, Math.round(effectiveFrameCount ?? layer.frameCount)), }; - if (layer.effectiveFrameCount !== undefined) { - metrics.overlayEffectiveFrames = Math.max(1, Math.round(layer.effectiveFrameCount)); + if (effectiveFrameCount !== undefined) { + metrics.overlayEffectiveFrames = Math.max(1, Math.round(effectiveFrameCount)); } return metrics; } @@ -1139,8 +1242,11 @@ export function resolveNvidiaCudaTiledOverlaySidecarSummaryMetrics( /** * Resolves the measured native encode FPS reported by the helper: the flush - * span measuredFps is authoritative, with the encode-loop fps as the backward- - * compatible fallback. Never derives FPS from wall-clock estimates. + * span measuredFps is authoritative, with the helper's encode-loop fps as the + * backward-compatible fallback. Never derives FPS from wall-clock estimates + * and never falls back to the configured output stream fps (summary.fps) and + * labels it as measured encode speed; if the helper reported no measured FPS + * this returns undefined. */ export function resolveNvidiaCudaNativeFps( summary: NvidiaCudaExportSummary | undefined, @@ -1231,32 +1337,39 @@ export function validateNvidiaCudaStageMetricInvariants( `temporalBlurStationaryFrames ${stationaryFrames} exceeds temporalBlurFrames ${temporalBlurFrames}`, ); } + // temporalBlurBgCacheBuilds count cache allocations/segments, while hits and + // temporalBlurBgPrecomposedFrames count per-frame background reuse. Builds are + // therefore NOT part of the frame budget and must never be summed into it; + // adding them produced a false positive (e.g. builds 1 + hits 4 vs. 4 + // precomposed frames). Each counter must stay finite/non-negative, and hits + // must not exceed the precomposed (or total temporal) frame budget. const bgCacheBuilds = native.temporalBgCacheBuilds; + if (typeof bgCacheBuilds === "number" && Number.isFinite(bgCacheBuilds) && bgCacheBuilds < 0) { + issues.push(`temporalBgCacheBuilds ${bgCacheBuilds} must be non-negative`); + } + const bgCacheHits = native.temporalBgCacheHits; - if ( - typeof bgCacheBuilds === "number" && - Number.isFinite(bgCacheBuilds) && - typeof bgCacheHits === "number" && - Number.isFinite(bgCacheHits) - ) { - const bgCacheTotal = bgCacheBuilds + bgCacheHits; - const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; - if ( - typeof bgPrecomposedFrames === "number" && - Number.isFinite(bgPrecomposedFrames) && - bgCacheTotal > bgPrecomposedFrames - ) { - issues.push( - `temporalBgCacheBuilds ${bgCacheBuilds} + temporalBgCacheHits ${bgCacheHits} exceed temporalBlurBgPrecomposedFrames ${bgPrecomposedFrames}`, - ); - } else if ( - typeof temporalBlurFrames === "number" && - Number.isFinite(temporalBlurFrames) && - bgCacheTotal > temporalBlurFrames - ) { - issues.push( - `temporalBgCacheBuilds ${bgCacheBuilds} + temporalBgCacheHits ${bgCacheHits} exceed temporalBlurFrames ${temporalBlurFrames}`, - ); + if (typeof bgCacheHits === "number" && Number.isFinite(bgCacheHits)) { + if (bgCacheHits < 0) { + issues.push(`temporalBgCacheHits ${bgCacheHits} must be non-negative`); + } else { + const bgPrecomposedFrames = native.temporalBlurBgPrecomposedFrames; + const hasBgPrecomposedFrames = + typeof bgPrecomposedFrames === "number" && Number.isFinite(bgPrecomposedFrames); + const cacheBudget: number | null = hasBgPrecomposedFrames + ? bgPrecomposedFrames + : typeof temporalBlurFrames === "number" && Number.isFinite(temporalBlurFrames) + ? temporalBlurFrames + : null; + if (cacheBudget !== null && bgCacheHits > cacheBudget) { + issues.push( + `temporalBgCacheHits ${bgCacheHits} exceeds ${ + hasBgPrecomposedFrames + ? "temporalBlurBgPrecomposedFrames" + : "temporalBlurFrames" + } ${cacheBudget}`, + ); + } } } @@ -1794,7 +1907,10 @@ export function mapNvidiaCudaWrapperProgressPercentage(progress: NativeStaticLay // inclusive) estimate. Only averageFps/instantFps reported by the native helper // count as measured encode speed; the frames/wall-clock estimate since process // spawn must be surfaced separately so callers never mistake it for encode -// throughput. +// throughput. Finalizing-stage progress must never emit the preparation- +// inclusive estimate: by then the helper has already measured encode speed on +// earlier progress lines, and the flush/mux span has no frame rate of its own, +// so an estimate there would only misrepresent the display. export function resolveNativeStaticLayoutFpsFields( progress: NativeStaticLayoutExportProgress, elapsedMs: number, @@ -1816,8 +1932,9 @@ export function resolveNativeStaticLayoutFpsFields( ? progress.instantFps : undefined; const hasNativeMeasured = nativeInstantFps !== undefined || nativeAverageFps !== undefined; + const finalizing = progress.stage === "finalizing"; const estimatedFps = - !hasNativeMeasured && elapsedMs > 0 && progress.currentFrame > 0 + !hasNativeMeasured && !finalizing && elapsedMs > 0 && progress.currentFrame > 0 ? (progress.currentFrame * 1000) / elapsedMs : undefined; return { @@ -1849,6 +1966,57 @@ export function hasNativeStaticLayoutProgressAdvanced( return progress.stage === "finalizing" && previous.stage !== "finalizing"; } +// Display-only preparation percentages for the NVIDIA CUDA startup substates. +// All stay inside the preparing window (<=3) so the renderer treats every +// event as non-rendering preparation and never derives an encode FPS from it. +// They are additive progression markers only; encode speed is reported later +// by the measured helper PROGRESS lines (nativeFps) and never here. +const NATIVE_STATIC_LAYOUT_PREPARE_SUBSTATE_PERCENTAGE: Readonly< + Record +> = { + "encoder-probe": 0.4, + "source-validation": 1.0, + "wrapper-launch": 1.6, + "cuda-nvenc-init": 2.2, + "first-frame": 2.8, +}; + +/** + * Builds an additive CUDA-preparation progress payload for a selected NVIDIA + * CUDA compositor route. The backend is labelled "nvidia-cuda-compositor" + * from the very first preparation event so the UI names the correct encoder + * before any renderer frame is produced. currentFrame stays 0 and percentage + * stays within the preparing window: this payload carries no FPS fields and + * must never be mistaken for measured encode speed (it is display-only). + */ +export function buildNvidiaCudaPrepareProgress( + sessionId: string | undefined, + substate: NativeStaticLayoutPrepareSubstate, + totalFrames: number, + elapsedMs: number, +): NativeStaticLayoutExportProgress { + return { + sessionId, + stage: "preparing", + substate, + backend: "nvidia-cuda-compositor", + currentFrame: 0, + totalFrames: Math.max(1, Math.floor(totalFrames)), + percentage: NATIVE_STATIC_LAYOUT_PREPARE_SUBSTATE_PERCENTAGE[substate], + elapsedMs: Math.max(0, Math.round(elapsedMs)), + }; +} + +function emitNvidiaCudaPrepareProgress( + onProgress: ((progress: NativeStaticLayoutExportProgress) => void) | undefined, + sessionId: string | undefined, + substate: NativeStaticLayoutPrepareSubstate, + totalFrames: number, + elapsedMs: number, +) { + onProgress?.(buildNvidiaCudaPrepareProgress(sessionId, substate, totalFrames, elapsedMs)); +} + function startNativeStaticLayoutExportPowerGuard() { try { const blockerId = powerSaveBlocker.start("prevent-app-suspension"); @@ -1861,7 +2029,11 @@ function startNativeStaticLayoutExportPowerGuard() { }, }; } catch (error) { - console.warn("[native-static-layout-export] Failed to start power guard", error); + console.warn( + formatLogTs(), + "[native-static-layout-export] Failed to start power guard", + error, + ); return { started: false, release: () => undefined, @@ -1878,7 +2050,11 @@ function setNativeStaticLayoutExportProcessPriority(pid: number | undefined, lab os.setPriority(pid, NATIVE_EXPORT_HIGH_PRIORITY); return true; } catch (error) { - console.warn(`[native-static-layout-export] Failed to raise ${label} priority`, error); + console.warn( + formatLogTs(), + `[native-static-layout-export] Failed to raise ${label} priority`, + error, + ); return false; } } @@ -2184,6 +2360,12 @@ async function runFfmpegWithMetrics( }); if (session) { session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below (killing an already-exited child on Windows emits an unhandled + // 'error' event when no listener is attached yet). + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -2329,6 +2511,11 @@ async function runFfmpegAudioMux( }); if (session) { session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below; see the CUDA wrapper spawn for the rationale. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -2831,11 +3018,6 @@ export function hasNvidiaGpuDeviceInGpuInfo(gpuInfo: unknown) { return Array.isArray(devices) && devices.some(isNvidiaGpuDevice); } -async function hasNvidiaGpuForCudaExportCandidate() { - const hasNvidiaGpu = await probeNvidiaGpuForCudaExportCandidate(); - return hasNvidiaGpu ?? true; -} - async function probeNvidiaGpuForCudaExportCandidate(): Promise { const getGPUInfo = ( app as typeof app & { @@ -2984,6 +3166,143 @@ export function getNativeGpuCompositorStallTimeoutMs() { return DEFAULT_NATIVE_GPU_STALL_TIMEOUT_MS; } +/** + * Session-scoped cache for the expensive NVIDIA CUDA availability probes + * (enumerating the helper wrapper candidates via fs.access and inspecting GPU + * info via Electron's getGPUInfo). Both the capabilities query + * (getNativeExportCapabilities) and the export route decision + * (getExperimentalNvidiaCudaExportSkipReason) share these values so the probes + * run once per session instead of once per call. The cache is keyed on a + * signature of the environment overrides and resolved app paths; whenever a + * relevant override or the resolved helper path changes the entry is rebuilt. + * A runtime helper failure is never promoted into availability: this cache only + * records the pre-flight probe results and is bypassed by the actual runtime + * wrapper invocation (runExperimentalNvidiaCudaStaticLayoutExport), so strict + * HEVC Hardware CUDA-only hard-fail behavior is unchanged. + */ +type NvidiaCudaAvailabilityCache = { + signature: string; + wrapperPath: string | null; + gpuAvailability: boolean | null; + capability: NativeExportCapabilities["nvidiaCuda"]; +}; + +let nvidiaCudaAvailabilityCache: NvidiaCudaAvailabilityCache | null = null; + +/** + * Resets the NVIDIA CUDA availability cache. Exposed for tests; the cache is + * session-scoped so resetting it forces the next capability/route query to + * re-probe the wrapper and GPU. + */ +export function resetNvidiaCudaAvailabilityCache() { + nvidiaCudaAvailabilityCache = null; +} + +/** + * Strict HEVC Hardware policy: when HEVC Hardware is requested the generalized + * NVIDIA CUDA compositor is the ONLY acceptable route. A non-zero + * shouldTryNvidiaCuda here hard-fails rather than falling back to the renderer + * raw path, Breeze, or CPU. Returns the error message to throw, or null when + * the strict guard does not apply. + */ +export function resolveNvidiaCudaStrictHevcHardFail( + requiresStrictHevcCuda: boolean, + shouldTryNvidiaCuda: boolean, + nvidiaCudaSkipReason: string | null, +): string | null { + if (!requiresStrictHevcCuda || shouldTryNvidiaCuda) { + return null; + } + return `HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (${nvidiaCudaSkipReason ?? "cursor-atlas-unavailable"}) (noCpuFallback:true)`; +} + +function getNvidiaCudaAvailabilityEnvSignature() { + const resourcesPath = ( + process as NodeJS.Process & { + resourcesPath?: string; + } + ).resourcesPath; + return JSON.stringify([ + process.platform, + process.env[NVIDIA_CUDA_EXPORT_ENV] ?? null, + process.env[NVIDIA_CUDA_ALLOW_AUDIO_EXPORT_ENV] ?? null, + process.env[NVIDIA_CUDA_FORCE_VIDEO_ONLY_ENV] ?? null, + process.env.RECORDLY_NVIDIA_CUDA_EXPORT_SCRIPT ?? null, + process.env.RECORDLY_NVIDIA_CUDA_NODE_EXE ?? null, + resourcesPath ?? null, + process.cwd(), + app.getAppPath(), + ]); +} + +function resolveNvidiaCudaCapability( + wrapperPath: string | null, + gpuAvailability: boolean | null, +): NativeExportCapabilities["nvidiaCuda"] { + const explicitEnabled = isExplicitNvidiaCudaExportEnabled(); + const explicitDisabled = isExplicitNvidiaCudaExportDisabled(); + // An inconclusive GPU probe (null) must NOT report the CUDA route + // unavailable: Electron's getGPUInfo can fail in dev/packaged runs while the + // live CUDA helper builds and initializes fine. The runtime attempt is the + // authoritative check (the helper fails with noCpuFallback on non-NVIDIA + // hardware), matching getExperimentalNvidiaCudaExportSkipReason which also + // treats an inconclusive probe as "let the helper decide". + const skipReason = explicitDisabled + ? "env-disabled" + : !wrapperPath + ? "cuda-wrapper-unavailable" + : gpuAvailability === false + ? "nvidia-gpu-unavailable" + : null; + if (gpuAvailability === null && !explicitDisabled && wrapperPath) { + console.info( + formatLogTs(), + "[native-export] NVIDIA CUDA GPU probe was inconclusive; letting the live helper decide at runtime", + { wrapperPath, reason: skipReason }, + ); + } else if (skipReason) { + console.info(formatLogTs(), "[native-export] NVIDIA CUDA availability", { + available: false, + skipReason, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + }); + } else { + console.info(formatLogTs(), "[native-export] NVIDIA CUDA availability", { + available: true, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + }); + } + + return { + available: skipReason === null, + skipReason, + hasNvidiaGpu: gpuAvailability, + hasWrapper: Boolean(wrapperPath), + explicitEnabled, + explicitDisabled, + userOptInRequired: !explicitEnabled, + }; +} + +async function ensureNvidiaCudaAvailabilityResolved(): Promise { + const signature = getNvidiaCudaAvailabilityEnvSignature(); + if (nvidiaCudaAvailabilityCache?.signature === signature) { + return nvidiaCudaAvailabilityCache; + } + + const wrapperPath = await resolveExperimentalNvidiaCudaExportScriptPath(); + const gpuAvailability = await probeNvidiaGpuForCudaExportCandidate(); + nvidiaCudaAvailabilityCache = { + signature, + wrapperPath, + gpuAvailability, + capability: resolveNvidiaCudaCapability(wrapperPath, gpuAvailability), + }; + return nvidiaCudaAvailabilityCache; +} + export async function getExperimentalNvidiaCudaExportSkipReason( options: NativeStaticLayoutExportOptions, ) { @@ -3003,10 +3322,11 @@ export async function getExperimentalNvidiaCudaExportSkipReason( } if (userOptIn) { - if (!(await resolveExperimentalNvidiaCudaExportScriptPath())) { + const cache = await ensureNvidiaCudaAvailabilityResolved(); + if (!cache.wrapperPath) { return "cuda-wrapper-unavailable"; } - if (!(await hasNvidiaGpuForCudaExportCandidate())) { + if ((cache.gpuAvailability ?? true) === false) { return "nvidia-gpu-unavailable"; } } @@ -3033,55 +3353,118 @@ export async function getNativeExportCapabilities(): Promise boolean; +} + +export interface NativeExportPrewarmOutcome { + sourceMetadataCached: boolean; + cudaAvailabilityResolved: boolean; + resolvedEncoders: string[]; + skipReasons: string[]; +} + +/** + * Deterministic, side-effect-free prewarming of the session-scoped caches used + * by native static-layout export: the validated source metadata probe cache and + * the NVIDIA CUDA availability cache. Encoder capability resolution is derived + * purely from the requested high-level codec/preference via + * getNativeEncoderCandidates. This never creates source proxies, output files, + * helper exports, or persists derived encoder names/settings. Any failure is + * diagnostics-only and must never poison availability: normal export probing and + * fallback stay unchanged. Strict HEVC Hardware CUDA-only hard-fail behavior is + * untouched because the availability cache only records pre-flight probe results + * and the live runtime export still validates its own route. + */ +export async function prewarmNativeExportCaches( + context: NativeExportPrewarmContext, +): Promise { + const skipReasons: string[] = []; + const outcome: NativeExportPrewarmOutcome = { + sourceMetadataCached: false, + cudaAvailabilityResolved: false, + resolvedEncoders: [], + skipReasons, }; + + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + + // Canonical source stat/identity + validated source metadata probe, reusing + // the existing exact-keyed bounded probe cache (identity + codec + encoding + // mode + encoder preference). Best-effort warm with the persisted export + // encoding mode (the same key a real export resolves for its source + // metadata), so a later export with the matching route hits the cache + // instead of re-probing. A later export with a different mode still re-probes + // because the exact cache key will not match. + try { + const ffmpegPath = getFfmpegBinaryPath(); + const metadata = await resolveNativeStaticLayoutSourceMetadata( + ffmpegPath, + { inputPath: context.inputPath, encodingMode: context.encodingMode }, + context.videoCodec, + context.encoderPreference, + ); + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + outcome.sourceMetadataCached = isNativeStaticLayoutSourceProbeCacheable(metadata); + skipReasons.push( + outcome.sourceMetadataCached + ? `source-metadata-cached:${metadata.codec}` + : "source-metadata-uncacheable", + ); + } catch { + // Diagnostics-only: a failed metadata probe never poisons availability and + // never blocks the response. Normal export probing is unchanged. + skipReasons.push("source-metadata-unavailable"); + } + + // Encoder capability resolution for the persisted/high-level codec and + // preference. Pure deterministic derivation (no I/O), so this is cheap. + outcome.resolvedEncoders = getNativeEncoderCandidates( + context.videoCodec, + context.encoderPreference, + process.platform, + ); + + // NVIDIA CUDA availability cache (only resolvable on platforms that ship the + // helper). The availability cache records pre-flight probe results only; a + // prewarm failure is never promoted into availability. + try { + const capabilities = await getNativeExportCapabilities(); + if (context.isSuperseded?.()) { + skipReasons.push("superseded"); + return outcome; + } + if (capabilities.nvidiaCuda.available) { + outcome.cudaAvailabilityResolved = true; + } else { + skipReasons.push(`cuda-unavailable:${capabilities.nvidiaCuda.skipReason ?? "unknown"}`); + } + } catch { + skipReasons.push("cuda-probe-failed"); + } + + return outcome; } export async function resolveExperimentalNvidiaCudaExportScriptPath() { @@ -3649,13 +4032,197 @@ async function prepareWindowsGpuWebcamInput( return { inputPath: outputPath, elapsedMs: result.elapsedMs }; } +type NativeStaticLayoutSourceIdentity = { + canonicalPath: string; + device: number; + inode: number; + size: number; + mtimeMs: number; + ctimeMs: number; +}; + +/** + * A validated native source probe recorded after a successful FFmpeg metadata + * probe. The entry is only reusable when the file identity (canonical path, + * device/inode, size, mtime/ctime) AND the route requirements (requested + * output codec, encoding mode, encoder preference) match exactly. It is never + * keyed by path alone and never reused across a changed/mutated source. + */ +export interface NativeStaticLayoutSourceProbeCacheEntry { + identity: NativeStaticLayoutSourceIdentity; + requestedCodec: ExportVideoCodec; + encodingMode: NativeExportEncodingMode; + encoderPreference: ExportEncoderPreference; + metadata: NativeVideoMetadataProbe; +} + +const NATIVE_STATIC_LAYOUT_SOURCE_PROBE_CACHE_MAX = 8; +let nativeStaticLayoutSourceProbeCache = new Map(); + +/** + * Resets the bounded native source probe cache. Exposed for tests; the cache is + * session-scoped so resetting it forces the next source preparation to re-probe + * the source with FFmpeg. + */ +export function resetNativeStaticLayoutSourceProbeCache() { + nativeStaticLayoutSourceProbeCache.clear(); +} + +function buildNativeStaticLayoutSourceIdentity(stat: { + dev: number | bigint; + ino: number | bigint; + size: number | bigint; + mtimeMs: number; + ctimeMs: number; +}): NativeStaticLayoutSourceIdentity | null { + const device = Number(stat.dev); + const inode = Number(stat.ino); + // A missing/unreliable identity (e.g. zeroed device/inode) must bypass the + // cache entirely and re-probe rather than risk a false reuse. + if ( + !Number.isSafeInteger(device) || + !Number.isSafeInteger(inode) || + device === 0 || + inode === 0 + ) { + return null; + } + return { + canonicalPath: "", + device, + inode, + size: Number(stat.size), + mtimeMs: stat.mtimeMs, + ctimeMs: stat.ctimeMs, + }; +} + +function isNativeStaticLayoutSourceProbeCacheable(metadata: NativeVideoMetadataProbe) { + const codec = (metadata.codec ?? "").trim().toLowerCase(); + return codec !== "" && codec !== "unknown"; +} + +/** + * Deterministic exact-match predicate for reusing a previous successful source + * probe. Returns true only when canonical path, device/inode, size, mtime/ctime, + * requested output codec, encoding mode, and encoder preference all match. Any + * mismatch forces a re-probe (or fail closed when identity is missing). + */ +export function canReuseNativeStaticLayoutSourceProbe( + entry: NativeStaticLayoutSourceProbeCacheEntry | undefined, + current: NativeStaticLayoutSourceIdentity & { + requestedCodec: ExportVideoCodec; + encodingMode: NativeExportEncodingMode; + encoderPreference: ExportEncoderPreference; + }, +): boolean { + if (!entry) { + return false; + } + if (!isNativeStaticLayoutSourceProbeCacheable(entry.metadata)) { + return false; + } + return ( + entry.identity.canonicalPath === current.canonicalPath && + entry.identity.device === current.device && + entry.identity.inode === current.inode && + entry.identity.size === current.size && + entry.identity.mtimeMs === current.mtimeMs && + entry.identity.ctimeMs === current.ctimeMs && + entry.requestedCodec === current.requestedCodec && + entry.encodingMode === current.encodingMode && + entry.encoderPreference === current.encoderPreference + ); +} + +function rememberNativeStaticLayoutSourceProbe( + canonicalPath: string, + entry: NativeStaticLayoutSourceProbeCacheEntry, +) { + nativeStaticLayoutSourceProbeCache.set(canonicalPath, entry); + // Bound the cache; evict the oldest entry (Map preserves insertion order). + if (nativeStaticLayoutSourceProbeCache.size > NATIVE_STATIC_LAYOUT_SOURCE_PROBE_CACHE_MAX) { + const oldestKey = nativeStaticLayoutSourceProbeCache.keys().next().value; + if (oldestKey !== undefined) { + nativeStaticLayoutSourceProbeCache.delete(oldestKey); + } + } +} + +/** + * Resolves the source metadata for native static-layout source preparation, + * reusing a recent validated probe only when the file identity and the route + * requirements match exactly. Misses, mutations, changed settings, missing + * identity, uncacheable/unknown codecs, and probe failures all re-probe with + * FFmpeg (or propagate the failure); nothing is ever trusted by path alone. + */ +async function resolveNativeStaticLayoutSourceMetadata( + ffmpegPath: string, + options: Pick, + requestedCodec: ExportVideoCodec, + encoderPreference: ExportEncoderPreference, +): Promise { + const inputPath = options.inputPath; + let canonicalPath: string | null = null; + let identity: NativeStaticLayoutSourceIdentity | null = null; + try { + const stat = await fs.stat(inputPath); + const identityBase = buildNativeStaticLayoutSourceIdentity(stat); + if (identityBase) { + // Stat and realpath race minimally, but both derive from the same file; + // any mutation between them is caught on the size/mtime/ctime match. + canonicalPath = await fs.realpath(inputPath).catch(() => inputPath); + identity = { ...identityBase, canonicalPath }; + } + } catch { + // Unreadable source or missing identity: never reuse a stale entry; fall + // through to a fresh probe which will surface the real error. + identity = null; + canonicalPath = null; + } + + if (identity && canonicalPath) { + const candidate = nativeStaticLayoutSourceProbeCache.get(canonicalPath); + if ( + candidate && + canReuseNativeStaticLayoutSourceProbe(candidate, { + ...identity, + requestedCodec, + encodingMode: options.encodingMode, + encoderPreference, + }) + ) { + return candidate.metadata; + } + } + + const metadata = await probeNativeVideoMetadata(ffmpegPath, options.inputPath); + if (identity && canonicalPath && isNativeStaticLayoutSourceProbeCacheable(metadata)) { + rememberNativeStaticLayoutSourceProbe(canonicalPath, { + identity, + requestedCodec, + encodingMode: options.encodingMode, + encoderPreference, + metadata, + }); + } + return metadata; +} + async function prepareNativeStaticLayoutSourceInput( ffmpegPath: string, options: NativeStaticLayoutExportOptions, outputPath: string, session: NativeStaticLayoutExportSession, + requestedCodec: ExportVideoCodec, + encoderPreference: ExportEncoderPreference, ) { - const metadata = await probeNativeVideoMetadata(ffmpegPath, options.inputPath); + const metadata = await resolveNativeStaticLayoutSourceMetadata( + ffmpegPath, + options, + requestedCodec, + encoderPreference, + ); if (!shouldCreateNativeStaticLayoutSourceProxy(metadata, options.inputPath)) { return { inputPath: options.inputPath, @@ -3697,26 +4264,44 @@ async function prepareNativeStaticLayoutSourceInput( } export function buildNativeStaticLayoutOverlayManifest( - layers: readonly NativeStaticLayoutOverlayLayer[], + layers: readonly (NativeStaticLayoutOverlayLayer | NativeCursorSpriteOverlayLayer)[], ) { return { layers: [...layers] .sort((left, right) => left.order - right.order || left.id.localeCompare(right.id)) - .map((layer) => ({ - id: layer.id, - path: layer.path, - x: layer.x, - y: layer.y, - width: layer.width, - height: layer.height, - // frameCount stays the logical output duration; effectiveFrameCount - // is the physical frame count the renderer wrote when identical-suffix - // dedup truncated the sidecar. Absent when every frame differs. - frameCount: layer.frameCount, - ...(layer.effectiveFrameCount !== undefined - ? { effectiveFrameCount: layer.effectiveFrameCount } - : {}), - })), + .map((layer) => + isCursorSpriteOverlayLayer(layer) + ? { + // A cursor-sprite layer is a packed RGBA frame strip whose + // per-frame top-left position comes from a JSON positions + // sidecar. Base x/y are always 0; order keeps it topmost. + id: layer.id, + kind: layer.kind, + order: layer.order, + path: layer.path, + positionsPath: layer.positionsPath, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + } + : { + id: layer.id, + path: layer.path, + x: layer.x, + y: layer.y, + width: layer.width, + height: layer.height, + // frameCount stays the logical output duration; effectiveFrameCount + // is the physical frame count the renderer wrote when identical- + // suffix dedup truncated the sidecar. Absent when every frame differs. + frameCount: layer.frameCount, + ...(layer.effectiveFrameCount !== undefined + ? { effectiveFrameCount: layer.effectiveFrameCount } + : {}), + }, + ), }; } @@ -3885,7 +4470,17 @@ export function buildExperimentalNvidiaCudaStaticLayoutArgs( if (options.zoomTelemetryPath) { args.push("--zoom-telemetry", options.zoomTelemetryPath); } - if (options.temporalBlur && options.temporalBlur.sampleCount >= 3) { + if (options.temporalBlur) { + // The renderer resolves temporal blur plans through + // getTemporalMotionBlurConfig, which clamps to at least + // TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT, so a plan below the minimum is + // an invariant violation. Reject it explicitly instead of silently + // dropping the effect through the sampleCount >= 3 gate below. + if (options.temporalBlur.sampleCount < TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT) { + throw new Error( + `unsupported-temporal-motion-blur: resolved temporal zoom motion blur plan uses ${options.temporalBlur.sampleCount} sample(s); the CUDA compositor minimum is ${TEMPORAL_MOTION_BLUR_MIN_SAMPLE_COUNT}. Refusing to silently drop the effect.`, + ); + } args.push( "--temporal-blur-sample-count", String(Math.round(options.temporalBlur.sampleCount)), @@ -3941,6 +4536,20 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( const workDir = path.join(chunkDirectory, "nvidia-cuda-work"); let effectiveOptions = options; if (options.overlayLayers?.length) { + // GPU-preparation audit: safe GPU-side composition (native cursor atlas, + // zoom/background composition, temporal zoom blur) already runs entirely + // in the generalized NVIDIA CUDA compositor with no renderer readback of + // video pixels. Browser raster overlays (captions, annotations, webcam, + // frame visuals) intentionally remain renderer-prepared transparent RGBA + // sidecar work: the compositor would need a live DOM/canvas rasterizer to + // draw arbitrary per-frame browser content natively, which is not + // supported, and direct canvas-to-NV12 transfer is not assumed until + // runtime support is proven (AGENTS.md native raw-frame transport). The + // renderer therefore bakes those layers into a bounded RGBA sidecar that + // the CUDA compositor uploads and alpha-blends on top of the composed, + // blurred video. These layers are export-session data only, never + // persisted, and a failed/incomplete sidecar preparation returns to the + // renderer raw-frame route rather than silently dropping a layer. const overlayManifestPath = path.join(chunkDirectory, "overlay-manifest.json"); await fs.writeFile( overlayManifestPath, @@ -4005,6 +4614,20 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( [pathKey]: `${ffmpegDirectory}${path.delimiter}${process.env[pathKey] ?? ""}`, }; const powerGuard = startNativeStaticLayoutExportPowerGuard(); + // A capability-only prewarm child may still hold a brief NVENC probe session; + // cancel it before this real export opens its own NVENC session so the two + // never contend for the GPU. Fire-and-forget: the prewarm was never awaited. + cancelInFlightCapabilityOnlyPrewarms(); + // Expected output frames; used to frame the display-only preparation + // substates (currentFrame is always 0 during preparation). + const prepareTotalFrames = Math.max(1, Math.ceil(options.durationSec * options.frameRate)); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "wrapper-launch", + prepareTotalFrames, + getNowMs() - startedAt, + ); return await new Promise<{ elapsedMs: number; @@ -4017,15 +4640,51 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( stdio: ["ignore", "pipe", "pipe"], windowsHide: true, }); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "cuda-nvenc-init", + prepareTotalFrames, + getNowMs() - startedAt, + ); const childPriorityApplied = setNativeStaticLayoutExportProcessPriority( child.pid, "NVIDIA CUDA export wrapper", ); - console.info("[native-static-layout-export] NVIDIA CUDA runtime guard started", { - childPriorityApplied, - powerGuardStarted: powerGuard.started, - }); + // Renderer-side overlay decision visibility: what the renderer actually + // prepared for this export (rgba full-canvas layers, tiled layers, or + // cursor-sprite ROI layers). This is the ground truth for diagnosing why a + // cursor-only export may have used the baked sidecar instead of the + // cursor-sprite fast path. + const overlayKinds = (options.overlayLayers ?? []).reduce>( + (acc, layer) => { + const kind = + "kind" in layer && layer.kind === NATIVE_CURSOR_SPRITE_LAYER_KIND + ? NATIVE_CURSOR_SPRITE_LAYER_KIND + : "rgba"; + acc[kind] = (acc[kind] ?? 0) + 1; + return acc; + }, + {}, + ); + console.info( + formatLogTs(), + "[native-static-layout-export] NVIDIA CUDA runtime guard started", + { + childPriorityApplied, + powerGuardStarted: powerGuard.started, + overlayLayerKinds: overlayKinds, + tiledOverlayLayers: options.tiledOverlayLayers?.length ?? 0, + }, + ); session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below (killing an already-exited child on Windows emits an unhandled + // 'error' event when no listener is attached yet). Real failures settle + // through the dedicated error/close handlers attached below. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -4034,6 +4693,7 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( let stderr = ""; let stderrLineBuffer = ""; let lastProgressPercentage = 0; + let firstFramePrepared = false; let lastProgressForStallGuard: { currentFrame: number; percentage: number; @@ -4079,10 +4739,25 @@ async function runExperimentalNvidiaCudaStaticLayoutExport( if (!progress) { continue; } + if (!firstFramePrepared) { + // The first helper PROGRESS line confirms the CUDA wrapper reached + // first-frame readiness (CUDA/NVENC initialized, encode producing + // frames). Emitted as a display-only preparing substate; it carries no + // FPS and does not overwrite the helper's measured encode rate. + firstFramePrepared = true; + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "first-frame", + prepareTotalFrames, + getNowMs() - startedAt, + ); + } const elapsedMs = Math.max(0, getNowMs() - startedAt); const fpsFields = resolveNativeStaticLayoutFpsFields(progress, elapsedMs); if (fpsFields.fpsSource === "estimated") { console.warn( + formatLogTs(), "[native-static-layout-export] Native helper has not reported measured encode FPS; using preparation-inclusive estimate", { backend: "nvidia-cuda-compositor", @@ -4227,6 +4902,11 @@ async function runExperimentalWindowsGpuStaticLayoutExport( stdio: ["ignore", "pipe", "pipe"], }); session.currentProcess = child; + // Swallow child-process errors that surface from the terminating kill + // below; see the CUDA wrapper spawn for the rationale. + child.on("error", () => { + /* handled by the dedicated handlers below */ + }); if (session.terminating) { child.kill("SIGKILL"); } @@ -4396,12 +5076,19 @@ export async function exportNativeStaticLayoutVideo( } if (options.overlayLayers?.length) { for (const layer of sortNativeStaticLayoutOverlayLayers(options.overlayLayers)) { - const validationError = validateNativeStaticLayoutOverlayLayer(layer, { - outputWidth: options.width, - outputHeight: options.height, - durationSec: options.durationSec, - frameRate: options.frameRate, - }); + const validationError = isCursorSpriteOverlayLayer(layer) + ? validateNativeCursorSpriteOverlayLayer(layer, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }) + : validateNativeStaticLayoutOverlayLayer(layer, { + outputWidth: options.width, + outputHeight: options.height, + durationSec: options.durationSec, + frameRate: options.frameRate, + }); if (validationError) { throw new Error(`Invalid native overlay layer: ${validationError}`); } @@ -4412,6 +5099,17 @@ export async function exportNativeStaticLayoutVideo( `Native overlay layer ${layer.id} is truncated: expected ${expectedBytes} bytes, received ${stat.size}`, ); } + if (isCursorSpriteOverlayLayer(layer)) { + // The cursor-sprite positions sidecar must exist so the native route + // never silently drops the cursor because the per-frame positions are + // missing (full JSON contents are validated by the native reader). + const positionsStat = await fs.stat(layer.positionsPath); + if (positionsStat.size <= 0) { + throw new Error( + `Native overlay layer ${layer.id} has an empty cursor-sprite positions file`, + ); + } + } } } if (options.tiledOverlayLayers?.length) { @@ -4454,13 +5152,60 @@ export async function exportNativeStaticLayoutVideo( "Cursor ownership by the native atlas requires the generalized NVIDIA CUDA compositor on Windows; the FFmpeg overlay route cannot draw the cursor and the overlay sidecar excluded it.", ); } + if ( + options.webcamNativeOwned === true && + !( + options.experimentalWindowsGpuCompositor && + process.platform === "win32" && + (options.experimentalNvidiaCudaExport === true || isExplicitNvidiaCudaExportEnabled()) + ) + ) { + // The renderer excluded webcam pixels from the overlay sidecar because + // the CUDA compositor owns the webcam natively. Only the generalized + // NVIDIA CUDA compositor can draw that webcam; the FFmpeg overlay route + // and the D3D11 helper cannot, so refusing the fallback is the only way + // to avoid silently dropping the webcam the sidecar excluded. + throw new Error( + "Webcam ownership by the native CUDA compositor requires the generalized NVIDIA CUDA compositor on Windows; the FFmpeg overlay route cannot draw the webcam and the overlay sidecar excluded it.", + ); + } + if (options.webcamNativeOwned === true && !options.webcamInputPath) { + throw new Error( + "Native webcam ownership requires a webcam input path; refusing to drop the webcam the overlay sidecar excluded.", + ); + } + if (options.webcamNativeOwned === true && (options.webcamSize ?? 0) <= 0) { + throw new Error( + "Native webcam ownership requires a positive webcam size; refusing to drop the webcam the overlay sidecar excluded.", + ); + } options = await normalizeNativeStaticLayoutBackground(options); - const nativeVideoEncoder = await resolveNativeVideoEncoder( - ffmpegPath, - options.encodingMode, - videoCodec, - encoderPreference, - ); + // The FFmpeg/rawvideo fallback needs a probed encoder; the NVIDIA CUDA and + // Windows-GPU compositor routes do not use it. Resolve it lazily, only on the + // first actual FFmpeg/raw fallback branch, so a CUDA-eligible native-layout + // job never spends time on the up-to-4x15s cold encoder probe BEFORE the CUDA + // route is selected/validated. The probe result is memoized for the session + // run (and internally cached by resolveNativeVideoEncoder), so the fallback + // branches only ever pay for it once. + let resolvedNativeVideoEncoder: string | null = null; + const ensureNativeVideoEncoder = async (): Promise => { + if (resolvedNativeVideoEncoder === null) { + resolvedNativeVideoEncoder = await resolveNativeVideoEncoder( + ffmpegPath, + options.encodingMode, + videoCodec, + encoderPreference, + ); + } + return resolvedNativeVideoEncoder; + }; + // Copies the given FFmpeg-args config with the probed encoder attached. Only + // the FFmpeg/raw fallback branches call this; CUDA/GPU routes never touch it. + const withNativeVideoEncoder = async ( + config: NativeStaticLayoutExportArgsConfig, + ): Promise => { + return { ...config, videoEncoder: await ensureNativeVideoEncoder() }; + }; if ( options.webcamInputPath && !(options.experimentalWindowsGpuCompositor && process.platform === "win32") @@ -4510,22 +5255,29 @@ export async function exportNativeStaticLayoutVideo( try { nativeStaticLayoutExportSessions.set(sessionId, session); + const exportRunStartedAt = getNowMs(); await fs.mkdir(chunkDirectory, { recursive: true }); const sourceInput = await prepareNativeStaticLayoutSourceInput( ffmpegPath, options, path.join(chunkDirectory, "source-proxy.mp4"), session, + videoCodec, + encoderPreference, ); if (sourceInput.elapsedMs > 0) { metrics.staticAssetExecMs = (metrics.staticAssetExecMs ?? 0) + sourceInput.elapsedMs; } if (sourceInput.inputPath !== options.inputPath) { - console.info("[native-static-layout-export] Prepared H.264 source proxy", { - sourceCodec: sourceInput.sourceCodec, - proxyCodec: sourceInput.proxyCodec, - elapsedMs: sourceInput.elapsedMs, - }); + console.info( + formatLogTs(), + "[native-static-layout-export] Prepared H.264 source proxy", + { + sourceCodec: sourceInput.sourceCodec, + proxyCodec: sourceInput.proxyCodec, + elapsedMs: sourceInput.elapsedMs, + }, + ); options = { ...options, inputPath: sourceInput.inputPath, @@ -4568,7 +5320,6 @@ export async function exportNativeStaticLayoutVideo( shadowIntensity: options.shadowIntensity, durationSec: options.durationSec, overlayLayers: options.overlayLayers, - videoEncoder: nativeVideoEncoder, }; const usePrecompositedLayout = shouldUsePrecompositedStaticLayout(options); let didRenderVideo = false; @@ -4665,6 +5416,7 @@ export async function exportNativeStaticLayoutVideo( nvidiaCudaForceVideoOnly: true, }; console.info( + formatLogTs(), "[native-static-layout-export] NVIDIA CUDA candidate will use shared audio mux validation", { audioMode: @@ -4684,6 +5436,7 @@ export async function exportNativeStaticLayoutVideo( nvidiaCudaSkipReason !== "env-disabled" ) { console.warn( + formatLogTs(), "[native-static-layout-export] Skipping NVIDIA CUDA compositor; falling back to Windows GPU compositor", { reason: nvidiaCudaSkipReason, @@ -4757,10 +5510,13 @@ export async function exportNativeStaticLayoutVideo( } } } - if (requiresStrictHevcCuda && !shouldTryNvidiaCuda) { - throw new Error( - `HEVC Hardware export requires the NVIDIA CUDA compositor; refusing fallback (${nvidiaCudaSkipReason ?? "cursor-atlas-unavailable"}) (noCpuFallback:true)`, - ); + const strictHevcHardFail = resolveNvidiaCudaStrictHevcHardFail( + requiresStrictHevcCuda, + shouldTryNvidiaCuda, + nvidiaCudaSkipReason, + ); + if (strictHevcHardFail) { + throw new Error(strictHevcHardFail); } if (shouldTryNvidiaCuda) { @@ -4768,6 +5524,31 @@ export async function exportNativeStaticLayoutVideo( const shouldMuxAudioInline = canMuxNvidiaCudaSourceAudioInline( experimentalNvidiaCudaOptions, ); + // The CUDA route is selected. Emit the two preparation substates that + // already completed before the wrapper launch (encoder capability + // probe and source validation). They are additive display-only + // progress with the NVIDIA CUDA compositor backend label from the + // very start; the wrapper-launch / cuda-nvenc-init / first-frame + // substates are emitted by the wrapper runner itself. + const prepareElapsed = () => Math.max(0, getNowMs() - exportRunStartedAt); + const prepareTotal = Math.max( + 1, + Math.ceil(options.durationSec * options.frameRate), + ); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "encoder-probe", + prepareTotal, + prepareElapsed(), + ); + emitNvidiaCudaPrepareProgress( + onProgress, + options.sessionId, + "source-validation", + prepareTotal, + prepareElapsed(), + ); const cudaResult = await runExperimentalNvidiaCudaStaticLayoutExport( ffmpegPath, experimentalNvidiaCudaOptions, @@ -4818,18 +5599,25 @@ export async function exportNativeStaticLayoutVideo( cudaResult.summary.nativeSummary, ); console.info( + formatLogTs(), "[native-static-layout-export] NVIDIA CUDA compositor completed", { elapsedMs: cudaResult.elapsedMs, - fps: cudaResult.summary.fps, + // summary.fps is the configured output frame rate (stream fps, + // never a measured encode throughput). It is labeled outputFps + // so it cannot be mistaken for measured encode speed, which is + // reported separately as nativeFps from the helper summary + // (resolveNvidiaCudaNativeFps). + outputFps: cudaResult.summary.fps, targetFrames: cudaResult.summary.targetFrames, durationSec: cudaResult.summary.durationSec, // timingsMs.nativeEncode is the full native helper-process wall // time (spawn to exit: source decode, layout composition, NVENC, - // flush). It is NOT the NVENC-API encode time, which is - // reported separately as nativeSummary.nvencMs. + // flush). It is NOT the NVENC-API encode time. The single explicit + // field is nativeEncodeWallMs; the low-level NVENC-API time is + // reported separately as nativeSummary.nvencMs (spread below via + // nativeSummaryMetrics), so no ambiguous duplicate is emitted. nativeEncodeWallMs: cudaResult.summary.timingsMs?.nativeEncode, - nativeEncodeMs: cudaResult.summary.timingsMs?.nativeEncode, muxMs: cudaResult.summary.timingsMs?.mux, endToEndMs: cudaResult.summary.timingsMs?.endToEnd, nativeFps: resolveNvidiaCudaNativeFps(cudaResult.summary), @@ -4924,6 +5712,15 @@ export async function exportNativeStaticLayoutVideo( `CUDA composition failed while temporal zoom motion blur is requested (${options.temporalBlur.sampleCount} samples); the fallback route cannot preserve temporal blur: ${error instanceof Error ? error.message : String(error)}`, ); } + if (options.webcamNativeOwned) { + // The renderer excluded webcam pixels from the overlay sidecar + // because the CUDA compositor owns the webcam natively; the + // remaining D3D11/FFmpeg fallback cannot draw it, so continuing + // would silently drop the webcam. + throw new Error( + `CUDA composition failed while the CUDA compositor owns the webcam (webcamNativeOwned); the fallback route cannot draw the webcam the overlay sidecar excluded: ${error instanceof Error ? error.message : String(error)}`, + ); + } } } @@ -4948,27 +5745,31 @@ export async function exportNativeStaticLayoutVideo( ); } await validateRenderedVideoOutput(); - console.info("[native-static-layout-export] Windows GPU compositor completed", { - elapsedMs: gpuResult.elapsedMs, - width: gpuResult.summary.width, - height: gpuResult.summary.height, - fps: gpuResult.summary.fps, - frames: gpuResult.summary.frames, - realtimeMultiplier: gpuResult.summary.realtimeMultiplier, - surfacePoolSize: gpuResult.summary.surfacePoolSize, - gpuDecodeSurface: gpuResult.summary.gpuDecodeSurface, - adapterIndex: gpuResult.summary.adapterIndex, - encoderBackend: gpuResult.summary.encoderBackend, - encoderTuningApplied: gpuResult.summary.encoderTuningApplied, - readMs: gpuResult.summary.readMs, - videoProcessMs: gpuResult.summary.videoProcessMs, - writeSampleMs: gpuResult.summary.writeSampleMs, - finalizeMs: gpuResult.summary.finalizeMs, - webcamOverlay: gpuResult.summary.webcamOverlay, - cursorOverlay: gpuResult.summary.cursorOverlay, - cursorAtlas: gpuResult.summary.cursorAtlas, - zoomOverlay: gpuResult.summary.zoomOverlay, - }); + console.info( + formatLogTs(), + "[native-static-layout-export] Windows GPU compositor completed", + { + elapsedMs: gpuResult.elapsedMs, + width: gpuResult.summary.width, + height: gpuResult.summary.height, + fps: gpuResult.summary.fps, + frames: gpuResult.summary.frames, + realtimeMultiplier: gpuResult.summary.realtimeMultiplier, + surfacePoolSize: gpuResult.summary.surfacePoolSize, + gpuDecodeSurface: gpuResult.summary.gpuDecodeSurface, + adapterIndex: gpuResult.summary.adapterIndex, + encoderBackend: gpuResult.summary.encoderBackend, + encoderTuningApplied: gpuResult.summary.encoderTuningApplied, + readMs: gpuResult.summary.readMs, + videoProcessMs: gpuResult.summary.videoProcessMs, + writeSampleMs: gpuResult.summary.writeSampleMs, + finalizeMs: gpuResult.summary.finalizeMs, + webcamOverlay: gpuResult.summary.webcamOverlay, + cursorOverlay: gpuResult.summary.cursorOverlay, + cursorAtlas: gpuResult.summary.cursorAtlas, + zoomOverlay: gpuResult.summary.zoomOverlay, + }, + ); const outputStat = await fs.stat(videoOnlyPath); metrics.chunkCount = 1; metrics.chunkDurationSec = options.durationSec; @@ -4988,11 +5789,29 @@ export async function exportNativeStaticLayoutVideo( if (session.terminating) { throw error; } + if (options.videoCodec === "hevc" && options.encoderPreference === "hardware") { + // Strict HEVC Hardware: the generalized NVIDIA CUDA compositor is the + // ONLY acceptable route. This outer GPU-block catch must never + // swallow the CUDA helper/noCpuFallback error (e.g. when there is no + // webcam, zoom telemetry, or native timeline) and attempt a full + // FFmpeg hevc_nvenc fallback before the renderer rejects it. Rethrow + // the original actionable error (which the inner CUDA catch already + // annotates with noCpuFallback:true and CUDA context) so no CPU, + // rawvideo, Breeze, or FFmpeg CUDA fallback path is reached. + const strictMessage = error instanceof Error ? error.message : String(error); + if (strictMessage.includes("noCpuFallback:true")) { + throw error; + } + throw new Error( + `HEVC Hardware NVIDIA CUDA compositor failed; refusing CPU, rawvideo, Breeze, or FFmpeg CUDA fallback (noCpuFallback:true): ${strictMessage}`, + ); + } if (hasNativeStaticLayoutTimeline(options)) { throw error; } metrics.fallbackChunkCount++; console.warn( + formatLogTs(), "[native-static-layout-export] Experimental Windows GPU compositor unavailable; falling back to FFmpeg static layout:", error, ); @@ -5017,6 +5836,15 @@ export async function exportNativeStaticLayoutVideo( `No GPU compositor produced output; the static-layout fallback cannot consume ${options.tiledOverlayLayers.length} tiled overlay layer(s) (tiled-overlays-unsupported-in-static-layout-fallback)`, ); } + if (!didRenderVideo && options.webcamNativeOwned) { + // The renderer excluded webcam pixels from the overlay sidecar because + // the CUDA compositor owns them. No fallback below (precomposited, + // FFmpeg CUDA overlay, or CPU pad) can draw that webcam, so continuing + // would silently drop it; fail fast instead. + throw new Error( + "No GPU compositor produced output while the CUDA compositor owns the webcam (webcamNativeOwned); the static-layout fallback cannot draw the webcam the overlay sidecar excluded.", + ); + } if (!didRenderVideo && usePrecompositedLayout) { const maskPath = path.join(chunkDirectory, "layout-mask.pgm"); @@ -5030,10 +5858,12 @@ export async function exportNativeStaticLayoutVideo( ), ); + const encoderConfig = await withNativeVideoEncoder(fullConfig); + const backgroundResult = await runFfmpegWithMetrics( ffmpegPath, buildNativeStaticBackgroundRenderArgs({ - ...fullConfig, + ...encoderConfig, inputPath: options.inputPath, outputPath: staticBackgroundPath, maskPath, @@ -5049,7 +5879,7 @@ export async function exportNativeStaticLayoutVideo( const fullResult = await runFfmpegWithMetrics( ffmpegPath, buildNativePrecompositedStaticLayoutArgs({ - ...fullConfig, + ...encoderConfig, staticBackgroundPath, maskPath, }), @@ -5073,9 +5903,10 @@ export async function exportNativeStaticLayoutVideo( outputBytes: outputStat.size, }); } else if (!didRenderVideo) { + const encoderConfig = await withNativeVideoEncoder(fullConfig); const primaryResult = await runFfmpegWithMetrics( ffmpegPath, - buildNativeCudaOverlayStaticLayoutArgs(fullConfig), + buildNativeCudaOverlayStaticLayoutArgs(encoderConfig), 15 * 60 * 1000, session, ); @@ -5098,7 +5929,7 @@ export async function exportNativeStaticLayoutVideo( metrics.fallbackChunkCount++; fullResult = await runFfmpegWithMetrics( ffmpegPath, - buildNativeCudaScaleCpuPadStaticLayoutArgs(fullConfig), + buildNativeCudaScaleCpuPadStaticLayoutArgs(encoderConfig), 15 * 60 * 1000, session, ); @@ -5144,7 +5975,7 @@ export async function exportNativeStaticLayoutVideo( offsetX: options.offsetX, offsetY: options.offsetY, backgroundColor: options.backgroundColor, - videoEncoder: nativeVideoEncoder, + videoEncoder: await ensureNativeVideoEncoder(), startSec: chunk.startSec, durationSec: chunk.durationSec, }; @@ -5256,12 +6087,21 @@ export async function exportNativeStaticLayoutVideo( if (!route) { throw new Error("Native static-layout export did not report a route"); } + // CUDA and D3D11 GPU-compositor routes do not use the FFmpeg encoder (it + // is resolved lazily, only on FFmpeg/raw fallback), so report the actual + // backend as the encoder name for those routes rather than a probed name. + const encoderName = + route === "nvidia-cuda-compositor" + ? "nvidia-cuda-compositor" + : route === "windows-d3d11-compositor" + ? "windows-d3d11-compositor" + : (resolvedNativeVideoEncoder ?? "cuda-static-composite"); return { outputPath: finalized.outputPath, metrics, videoCodec, encoderPreference, - encoderName: nativeVideoEncoder, + encoderName, route, }; } catch (error) { @@ -5359,11 +6199,20 @@ export async function probeNativeVideoEncoder( stderrOutput += chunk.toString(); }); + // Swallow child errors (e.g. EPIPE on stdin, or a SIGKILL on an already + // exited probe process). Without a listener these surface as uncaught + // "The process not found" noise on Windows. The close handler below + // settles the probe result. + process.on("error", () => { + /* probe failure settles via close; nothing to do here */ + }); + process.on("close", (code) => { clearTimeout(timeout); void removeTemporaryExportFile(outputPath); if (code !== 0 && stderrOutput.trim().length > 0) { console.warn( + formatLogTs(), `[native-export] Encoder probe failed for ${encoderName}:`, stderrOutput.trim(), ); @@ -5588,7 +6437,7 @@ export async function muxNativeVideoExportAudio( const ffmpegExecStartedAt = getNowMs(); await runFfmpegAudioMux(ffmpegPath, args, 15 * 60 * 1000, options, onProgress, session); metrics.ffmpegExecMs = getNowMs() - ffmpegExecStartedAt; - console.info("[native-video-export] Audio mux completed", { + console.info(formatLogTs(), "[native-video-export] Audio mux completed", { ffmpegExecMs: metrics.ffmpegExecMs, audioMode: options.audioMode, tempVideoBytes: metrics.tempVideoBytes, diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts index 16413c055..265f4b0e3 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts @@ -177,7 +177,7 @@ describe("planNativeStaticLayoutRoutes", () => { ); }); - it("falls back to rawvideo when HEVC CUDA is unavailable", () => { + it("hard-fails when HEVC CUDA is unavailable and Hardware is strict", () => { const plan = planNativeStaticLayoutRoutes({ videoCodec: "hevc", encoderPreference: "hardware", @@ -187,10 +187,16 @@ describe("planNativeStaticLayoutRoutes", () => { }); expect(plan.selectedRoute).toBeNull(); - expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.fallbackRoute).toBe("hard-fail"); + expect(plan.noCpuFallback).toBe(true); expect(plan.fallbackReason).toBe("hevc-hardware-route-unavailable:nvidia-gpu-unavailable"); expect(plan.decisions).toEqual( expect.arrayContaining([ + { + route: "nvidia-cuda-compositor", + status: "rejected", + reasons: ["nvidia-gpu-unavailable"], + }, { route: "windows-d3d11-compositor", status: "rejected", @@ -205,6 +211,21 @@ describe("planNativeStaticLayoutRoutes", () => { ); }); + it("keeps HEVC Auto rawvideo fallback non-strict", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: { ...cudaProbe, skipReason: "nvidia-gpu-unavailable" }, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBeNull(); + expect(plan.fallbackRoute).toBe("native-rawvideo"); + expect(plan.noCpuFallback).toBe(false); + expect(plan.fallbackReason).toBe("hevc-cuda-unavailable:nvidia-gpu-unavailable"); + }); + it("keeps CPU preference out of every GPU route", () => { const plan = planNativeStaticLayoutRoutes({ videoCodec: "h264", diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts index 92706e6b2..f7a19600c 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts @@ -8,7 +8,7 @@ export type NativeStaticLayoutRoute = | "nvidia-cuda-compositor" | "windows-d3d11-compositor" | "ffmpeg-static-layout"; -export type NativeStaticLayoutFallbackRoute = "native-rawvideo"; +export type NativeStaticLayoutFallbackRoute = "native-rawvideo" | "hard-fail"; export interface NativeStaticLayoutRouteDecision { route: NativeStaticLayoutRoute; @@ -54,6 +54,12 @@ export interface NativeStaticLayoutRoutePlan { selectedRoute: NativeStaticLayoutRoute | null; fallbackRoute: NativeStaticLayoutFallbackRoute | null; fallbackReason: string | null; + /** + * Strict HEVC Hardware policy: when true the export MUST hard-fail with an + * actionable error instead of falling back to renderer raw frames, Breeze, + * or CPU. No consumer may convert this plan into a rawvideo fallback. + */ + noCpuFallback: boolean; decisions: NativeStaticLayoutRouteDecision[]; cuda: NvidiaCudaExportCapabilityProbe; d3d11: WindowsD3D11ExportCapabilityProbe; @@ -68,14 +74,17 @@ function createRawVideoFallbackPlan(options: { source: NativeStaticLayoutRouteSource; reason: string; cudaReason: string; + noCpuFallback?: boolean; }) { const { videoCodec, encoderPreference, cuda, d3d11, source } = options; + const noCpuFallback = options.noCpuFallback === true; return { videoCodec, encoderPreference, selectedRoute: null, - fallbackRoute: "native-rawvideo" as const, + fallbackRoute: noCpuFallback ? "hard-fail" : "native-rawvideo", fallbackReason: options.reason, + noCpuFallback, decisions: [ { route: "nvidia-cuda-compositor" as const, @@ -150,6 +159,7 @@ export function planNativeStaticLayoutRoutes(options: { selectedRoute: "nvidia-cuda-compositor", fallbackRoute: null, fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, @@ -168,6 +178,9 @@ export function planNativeStaticLayoutRoutes(options: { ? `hevc-hardware-route-unavailable:${cuda.skipReason}` : `hevc-cuda-unavailable:${cuda.skipReason}`, cudaReason: cuda.skipReason, + // Strict HEVC Hardware policy: the NVIDIA CUDA compositor is the ONLY + // acceptable route. Never fall back to renderer rawvideo, Breeze, or CPU. + noCpuFallback: encoderPreference === "hardware", }); } @@ -207,6 +220,7 @@ export function planNativeStaticLayoutRoutes(options: { selectedRoute: "nvidia-cuda-compositor", fallbackRoute: null, fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, @@ -237,6 +251,7 @@ export function planNativeStaticLayoutRoutes(options: { selectedRoute: "windows-d3d11-compositor", fallbackRoute: null, fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, @@ -260,6 +275,7 @@ export function planNativeStaticLayoutRoutes(options: { selectedRoute: "ffmpeg-static-layout", fallbackRoute: null, fallbackReason: null, + noCpuFallback: false, decisions, cuda, d3d11, diff --git a/electron/ipc/nativeVideoExport.test.ts b/electron/ipc/nativeVideoExport.test.ts index d5e26f216..cc910b1a7 100644 --- a/electron/ipc/nativeVideoExport.test.ts +++ b/electron/ipc/nativeVideoExport.test.ts @@ -406,6 +406,75 @@ describe("native static layout command builders", () => { expect(filterComplex).toContain("[layout][overlay_0]overlay=x=0:y=0:format=auto"); }); + it("sorts shuffled overlay layers by order then id in the precomposited branch", () => { + const args = buildNativePrecompositedStaticLayoutArgs({ + ...baseConfig, + staticBackgroundPath: "background.png", + overlayLayers: [ + { + id: "caption", + order: 3, + path: "caption.rgba", + x: 0, + y: 800, + width: 1920, + height: 280, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "cursor", + order: 1, + path: "cursor.rgba", + x: 0, + y: 0, + width: 1920, + height: 1080, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + { + id: "annotation", + order: 2, + path: "annotation.rgba", + x: 0, + y: 540, + width: 1920, + height: 540, + frameRate: 60, + durationSec: 60, + frameCount: 3600, + pixelFormat: "rgba", + }, + ], + }); + const filterComplex = args[args.indexOf("-filter_complex") + 1]; + + expect(args).toEqual( + expect.arrayContaining([ + "-f", + "rawvideo", + "cursor.rgba", + "annotation.rgba", + "caption.rgba", + ]), + ); + expect(filterComplex).toContain("[2:v]format=rgba[overlay_0]"); + expect(filterComplex).toContain("[layout][overlay_0]overlay=x=0:y=0:format=auto"); + expect(filterComplex).toContain("[3:v]format=rgba[overlay_1]"); + expect(filterComplex).toContain( + "[layout_overlay_0][overlay_1]overlay=x=0:y=540:format=auto", + ); + expect(filterComplex).toContain("[4:v]format=rgba[overlay_2]"); + expect(filterComplex).toContain( + "[layout_overlay_1][overlay_2]overlay=x=0:y=800:format=auto", + ); + }); + it("splits long exports into bounded chunks", () => { expect(buildNativeStaticLayoutChunks(367.5, 120)).toEqual([ { index: 0, startSec: 0, durationSec: 120 }, diff --git a/electron/ipc/nativeVideoExport.ts b/electron/ipc/nativeVideoExport.ts index daa549c0b..0077a6a80 100644 --- a/electron/ipc/nativeVideoExport.ts +++ b/electron/ipc/nativeVideoExport.ts @@ -593,6 +593,9 @@ export function buildNativePrecompositedStaticLayoutArgs( const durationSec = formatFfmpegSeconds(Math.max(0.001, config.durationSec ?? 1) * 1000); const useMask = Boolean(config.maskPath && (config.borderRadius ?? 0) > 0.5); + const overlayLayers = [...(config.overlayLayers ?? [])].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); const args = ["-y", "-hide_banner", "-loglevel", "error"]; pushFfmpegTimeSliceArgs(args, config.startSec, config.durationSec); args.push( @@ -624,7 +627,7 @@ export function buildNativePrecompositedStaticLayoutArgs( config.maskPath, ); } - for (const layer of config.overlayLayers ?? []) { + for (const layer of overlayLayers) { args.push( "-f", "rawvideo", @@ -649,7 +652,7 @@ export function buildNativePrecompositedStaticLayoutArgs( ]; let currentLabel = "layout"; const firstOverlayInputIndex = useMask ? 3 : 2; - for (const [index, layer] of (config.overlayLayers ?? []).entries()) { + for (const [index, layer] of overlayLayers.entries()) { const nextLabel = `layout_overlay_${index}`; filterParts.push( `[${firstOverlayInputIndex + index}:v]format=rgba[overlay_${index}]`, diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index 0d8f9c32b..d7e70f1ed 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -15,6 +15,7 @@ import { } from "../export/exportStream"; import { attachNativeVideoExportFramePort, + cancelInFlightCapabilityOnlyPrewarms, closeNativeVideoExportFramePort, enqueueNativeVideoExportFrameWrite, enqueueNativeVideoExportFrameWrites, @@ -40,6 +41,7 @@ import { settleNativeVideoExportWriteFrameRequest, } from "../export/native-video"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { formatLogTs } from "../log"; import { buildNativeH264StreamExportArgs, buildNativeVideoExportArgs, @@ -66,6 +68,32 @@ function getPartialExportDestinationPath(destinationPath: string) { const MAX_IN_MEMORY_EXPORT_BYTES = 0x7fffffff; +/** + * Structured, timestamped route/settings summary logged at the `native-video- + * export-start` IPC boundary. Captures only high-level request settings (codec, + * encoder preference, input/mode, dimensions, frame rate) so operators can see + * the exact route requested before encoder resolution. Never includes media + * bytes, source paths, or runtime encoder names. + */ +function formatNativeExportRequestSettings(settings: { + videoCodec: ExportVideoCodec; + encoderPreference: ExportEncoderPreference; + inputMode: "rawvideo" | "h264-stream"; + encodingMode: NativeExportEncodingMode; + width: number; + height: number; + frameRate: number; +}) { + return ( + `codec=${settings.videoCodec} ` + + `preference=${settings.encoderPreference} ` + + `input=${settings.inputMode} ` + + `mode=${settings.encodingMode} ` + + `${settings.width}x${settings.height} ` + + `fps=${settings.frameRate}` + ); +} + function getInMemoryExportTooLargeMessage(byteLength: number) { if (byteLength <= MAX_IN_MEMORY_EXPORT_BYTES) { return null; @@ -296,18 +324,37 @@ export function registerExportHandlers() { encoderPreference?: ExportEncoderPreference; }, ) => { + const inputMode = options.inputMode ?? "rawvideo"; + const videoCodec = options.videoCodec ?? "h264"; + const encoderPreference = options.encoderPreference ?? "auto"; + let sessionId = ""; try { if (options.width % 2 !== 0 || options.height % 2 !== 0) { throw new Error("Native export requires even output dimensions"); } const ffmpegPath = getFfmpegBinaryPath(); - const inputMode = options.inputMode ?? "rawvideo"; - const videoCodec = options.videoCodec ?? "h264"; - const encoderPreference = options.encoderPreference ?? "auto"; - const sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + sessionId = `recordly-export-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; const outputPath = path.join(app.getPath("temp"), `${sessionId}.mp4`); + // Recommend the real route once an encoder has been resolved; the raw + // request line below is emitted before encoder resolution so a mismatch + // between the requested prewarm (e.g. hevc/hardware) and the encoder the + // export actually started with is immediately visible in one place. + const requestSettings = formatNativeExportRequestSettings({ + videoCodec, + encoderPreference, + inputMode, + encodingMode: options.encodingMode, + width: options.width, + height: options.height, + frameRate: options.frameRate, + }); + console.log( + formatLogTs(), + `[native-export] Start request session=${sessionId} ${requestSettings}`, + ); + let encoderName: string; let ffmpegArgs: string[]; @@ -336,6 +383,12 @@ export function registerExportHandlers() { ffmpegArgs = buildNativeVideoExportArgs(encoderName, options, outputPath); } + // A capability-only prewarm child may still hold a brief NVENC probe + // session; cancel it before this real export opens its real encoder + // session so they never contend for the GPU/hardware encoder. The + // prewarm is fire-and-forget (never awaited) so this never blocks IPC. + cancelInFlightCapabilityOnlyPrewarms(); + const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, { stdio: ["pipe", "ignore", "pipe"], }) as ChildProcessByStdio; @@ -366,7 +419,7 @@ export function registerExportHandlers() { framePortReady: false, nextFrameSequence: 0, pendingFrameRequests: new Map(), - completedFrameRequestIds: new Set(), + highestAcceptedFrameRequestId: -1, completionPromise: new Promise((resolve, reject) => { ffmpegProcess.once("error", (error) => { const processError = @@ -437,7 +490,8 @@ export function registerExportHandlers() { nativeVideoExportSessions.set(sessionId, session); console.log( - `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? "hardware" : "software"} session ${sessionId} with ${encoderName}`, + formatLogTs(), + `[native-export] Started ${isHardwareAcceleratedVideoEncoder(encoderName) ? "hardware" : "software"} session=${sessionId} encoder=${encoderName} route=${useH264StreamCopy ? "h264-stream-copy" : "native-raw"} ${requestSettings}`, ); return { @@ -446,8 +500,18 @@ export function registerExportHandlers() { encoderName, }; } catch (error) { + const failedRequestSettings = formatNativeExportRequestSettings({ + videoCodec, + encoderPreference, + inputMode, + encodingMode: options.encodingMode, + width: options.width, + height: options.height, + frameRate: options.frameRate, + }); console.error( - "[native-export] Failed to start native video export session:", + formatLogTs(), + `[native-export] Failed to start native video export session session=${sessionId || "unknown"} ${failedRequestSettings}:`, error, ); return { @@ -477,7 +541,7 @@ export function registerExportHandlers() { metadata, }; } catch (error) { - console.warn("[probe-native-video-metadata] Failed:", error); + console.warn(formatLogTs(), "[probe-native-video-metadata] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), @@ -492,7 +556,7 @@ export function registerExportHandlers() { capabilities: await getNativeExportCapabilities(), }; } catch (error) { - console.warn("[native-export-capabilities] Failed:", error); + console.warn(formatLogTs(), "[native-export-capabilities] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), @@ -537,7 +601,7 @@ export function registerExportHandlers() { metrics: result.metrics, }; } catch (error) { - console.warn("[native-static-layout-export] Failed:", error); + console.warn(formatLogTs(), "[native-static-layout-export] Failed:", error); return { success: false, error: error instanceof Error ? error.message : String(error), diff --git a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs index dc1889c04..8cc9a14cf 100644 --- a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs +++ b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs @@ -1367,6 +1367,21 @@ if (contentWidth > 0 && contentHeight > 0) { if (zoomTelemetry) { encodeArgs.push("--zoom-samples", resolve(zoomTelemetry)); } +if (temporalBlurSampleCount > 0) { + // The native compositor must advertise --temporal-blur-sample-count in its + // --help usage before the wrapper forwards the resolved temporal zoom + // motion blur plan (mirror of the tiled-overlay probe below). Until then + // temporal blur cannot be composited and the export fails fast instead of + // silently dropping the effect. + const nativeHelp = run(nativeProbe, ["--help"]).stdout; + if (!nativeHelp.includes("--temporal-blur-sample-count")) { + fail( + "unsupported-temporal-motion-blur: the native NVIDIA CUDA compositor does not support temporal zoom motion blur yet; " + + "main.cu must consume --temporal-blur-sample-count (this build) or the renderer must " + + "keep the effect on a CUDA-capable helper.", + ); + } +} if (temporalBlurSampleCount >= 3) { encodeArgs.push( "--temporal-blur-sample-count", @@ -1376,10 +1391,26 @@ if (temporalBlurSampleCount >= 3) { "--temporal-blur-weight-power", String(temporalBlurWeightPower), ); +} else if (temporalBlurSampleCount > 0) { + // The TS-side invariant rejects resolved plans below the minimum (3), but a + // direct wrapper invocation could still request 1-2 samples; never silently + // omit the effect without a diagnostic. + console.warn( + `[nvidia-cuda-export] Temporal zoom motion blur requested with ${temporalBlurSampleCount} sample(s), below the minimum of 3; omitting the effect (unsupported-temporal-motion-blur)`, + ); } -const overlayLayers = readOverlayManifest(overlayManifest, { outputWidth, outputHeight }); -if (overlayLayers.length) { - for (const layer of overlayLayers) { +// The manifest may mix fixed-position rgba layers and cursor-sprite layers. +// rgba layers keep the proven per-layer --overlay descriptor; cursor-sprite +// layers are forwarded to the native cursor-sprite compositor route that owns +// the packed frame strip + per-frame positions validation. +const overlayLayers = readOverlayManifest(overlayManifest, { + outputWidth, + outputHeight, +}); +const rgbaOverlayLayers = overlayLayers.filter((layer) => layer.kind === "rgba"); +const cursorSpriteLayers = overlayLayers.filter((layer) => layer.kind === "cursor-sprite"); +if (rgbaOverlayLayers.length) { + for (const layer of rgbaOverlayLayers) { encodeArgs.push( "--overlay", layer.path, @@ -1395,6 +1426,31 @@ if (overlayLayers.length) { ); } } +if (cursorSpriteLayers.length) { + const nativeHelp = run(nativeProbe, ["--help"]).stdout; + if (!nativeHelp.includes("--cursor-sprite")) { + fail( + "The native NVIDIA CUDA compositor does not support cursor-sprite overlays yet; " + + "main.cu must consume --cursor-sprite (this build) or the renderer must keep " + + "the baked cursor overlay sidecar fallback.", + ); + } + for (const layer of cursorSpriteLayers) { + // positions are validated/clamped on the JS side above; the native + // compositor re-validates the positions file and hard-fails (noCpuFallback) + // so the cursor is never silently omitted on a strict native route. + encodeArgs.push( + "--cursor-sprite", + layer.id, + String(layer.order), + layer.path, + resolve(layer.positionsPath), + String(layer.width), + String(layer.height), + String(layer.frameCount), + ); + } +} // Tiled/delta sparse overlay stream: the versioned descriptor was validated by // readTiledOverlayManifest (independently of the TS side). The native CUDA // compositor consumes the descriptor itself; it must advertise @@ -1649,7 +1705,7 @@ console.log( overlay: overlayLayers.length || tiledOverlayLayers.length ? { - layers: overlayLayers.map((layer) => ({ + layers: rgbaOverlayLayers.map((layer) => ({ id: layer.id, path: layer.path, x: layer.x, @@ -1666,6 +1722,20 @@ console.log( physicalFrameCount: layer.effectiveFrameCount ?? layer.frameCount, })), + cursorSprite: cursorSpriteLayers.length + ? { + layers: cursorSpriteLayers.map((layer) => ({ + id: layer.id, + order: layer.order, + path: layer.path, + positionsPath: layer.positionsPath, + width: layer.width, + height: layer.height, + frameCount: layer.frameCount, + positionsCount: layer.positions.length, + })), + } + : null, tiled: tiledOverlayLayers.length ? { layers: tiledOverlayMetrics } : null, diff --git a/src/components/video-editor/ExportSettingsMenu.tsx b/src/components/video-editor/ExportSettingsMenu.tsx index fca57b755..402738b4c 100644 --- a/src/components/video-editor/ExportSettingsMenu.tsx +++ b/src/components/video-editor/ExportSettingsMenu.tsx @@ -475,7 +475,12 @@ export function ExportSettingsMenu({ setBitrateDraft(event.target.value); const parsed = Number(event.target.value); if (Number.isFinite(parsed)) { - onExportBitrateMbpsChange?.(parsed); + onExportBitrateMbpsChange?.( + Math.min( + effectiveMaxMbps, + Math.max(EXPORT_BITRATE_MIN_MBPS, parsed), + ), + ); } }} onBlur={commitBitrateDraft} diff --git a/src/components/video-editor/editorPreferences.test.ts b/src/components/video-editor/editorPreferences.test.ts index 37b90fdd6..be05e77a6 100644 --- a/src/components/video-editor/editorPreferences.test.ts +++ b/src/components/video-editor/editorPreferences.test.ts @@ -501,7 +501,7 @@ describe("editorPreferences", () => { expect(normalized.exportVideoCodec).toBe("h264"); expect(normalized.exportEncoderPreference).toBe("auto"); expect(normalized.exportBitrateMode).toBe("auto"); - expect(normalized.exportBitrateMbps).toBe(200); + expect(normalized.exportBitrateMbps).toBe(105); expect(normalizeEditorPreferences({ exportBitrateMbps: -3 }).exportBitrateMbps).toBe(1); expect( @@ -544,7 +544,7 @@ describe("editorPreferences", () => { exportVideoCodec: "hevc", exportEncoderPreference: "cpu", exportBitrateMode: "custom", - exportBitrateMbps: 80, + exportBitrateMbps: 70, cropRegion: DEFAULT_CROP_REGION, autoCaptionSettings: DEFAULT_AUTO_CAPTION_SETTINGS, }, @@ -556,7 +556,7 @@ describe("editorPreferences", () => { exportVideoCodec: "hevc", exportEncoderPreference: "cpu", exportBitrateMode: "custom", - exportBitrateMbps: 80, + exportBitrateMbps: 70, }); }); }); diff --git a/src/components/video-editor/projectPersistence.test.ts b/src/components/video-editor/projectPersistence.test.ts index 93980ff36..edf843d9c 100644 --- a/src/components/video-editor/projectPersistence.test.ts +++ b/src/components/video-editor/projectPersistence.test.ts @@ -68,7 +68,14 @@ describe("normalizeProjectEditor", () => { expect(editor.exportVideoCodec).toBe("h264"); expect(editor.exportEncoderPreference).toBe("auto"); expect(editor.exportBitrateMode).toBe("auto"); - expect(editor.exportBitrateMbps).toBe(200); + expect(editor.exportBitrateMbps).toBe(105); + + const hevcClamp = normalizeProjectEditor({ + exportVideoCodec: "hevc", + exportBitrateMbps: 5000, + }); + expect(hevcClamp.exportVideoCodec).toBe("hevc"); + expect(hevcClamp.exportBitrateMbps).toBe(70); const lowClamp = normalizeProjectEditor({ exportBitrateMbps: -3 }); expect(lowClamp.exportBitrateMbps).toBe(1); diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 4d5762c65..be0129e16 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -242,8 +242,8 @@ export function normalizeExportBitrateMode(value: unknown): ExportBitrateMode { return "auto"; } -export function normalizeExportBitrateMbps(value: unknown): number { - return clampCustomBitrateMbps(typeof value === "number" ? value : Number.NaN); +export function normalizeExportBitrateMbps(value: unknown, codec?: ExportVideoCodec): number { + return clampCustomBitrateMbps(typeof value === "number" ? value : Number.NaN, codec); } function normalizeZoomTransitionEasing( @@ -930,6 +930,7 @@ export function normalizeProjectEditor(editor: Partial): Pro ); const normalizedMotionPreset = CURSOR_MOTION_PRESETS[resolveCursorMotionPresetId(normalizedMotionValues)]; + const normalizedExportVideoCodec = normalizeExportVideoCodec(editor.exportVideoCodec); return { wallpaper: typeof editor.wallpaper === "string" ? editor.wallpaper : DEFAULT_WALLPAPER_PATH, @@ -1128,10 +1129,13 @@ export function normalizeProjectEditor(editor: Partial): Pro ? editor.exportQuality : "source", mp4FrameRate: normalizeExportMp4FrameRate(editor.mp4FrameRate), - exportVideoCodec: normalizeExportVideoCodec(editor.exportVideoCodec), + exportVideoCodec: normalizedExportVideoCodec, exportEncoderPreference: normalizeExportEncoderPreference(editor.exportEncoderPreference), exportBitrateMode: normalizeExportBitrateMode(editor.exportBitrateMode), - exportBitrateMbps: normalizeExportBitrateMbps(editor.exportBitrateMbps), + exportBitrateMbps: normalizeExportBitrateMbps( + editor.exportBitrateMbps, + normalizedExportVideoCodec, + ), exportFormat: editor.exportFormat === "gif" ? "gif" : "mp4", gifFrameRate: editor.gifFrameRate === 15 || diff --git a/src/i18n/locales/de/settings.json b/src/i18n/locales/de/settings.json index fc0090162..0cf4003d8 100644 --- a/src/i18n/locales/de/settings.json +++ b/src/i18n/locales/de/settings.json @@ -248,7 +248,7 @@ "auto": "Automatisch", "custom": "Benutzerdefiniert", "mbpsInput": "Benutzerdefinierte Bitrate in Mbit/s", - "range": "1–200 Mbit/s" + "range": "1–105 Mbit/s (H.264) · 1–70 Mbit/s (HEVC)" }, "hevcHint": "HEVC (H.265) erzeugt kleinere Dateien, wird aber möglicherweise nicht von älteren oder Web-Playern unterstützt. Die Vorschauwiedergabe in Recordly bleibt unverändert.", "hardwareUnavailable": "Kein nutzbarer Hardware-Encoder gefunden. Wählen Sie CPU oder Auto, oder aktualisieren Sie den GPU-Treiber.", diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 0512feeb2..809481762 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -268,7 +268,7 @@ "auto": "Auto", "custom": "Custom", "mbpsInput": "Custom bitrate in Mbps", - "range": "1–200 Mbps" + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" }, "hardwareUnavailable": "No usable hardware encoder was found. Pick CPU or Auto, or update your GPU driver.", "errors": { diff --git a/src/i18n/locales/es/settings.json b/src/i18n/locales/es/settings.json index 8151f53a6..a026397c8 100644 --- a/src/i18n/locales/es/settings.json +++ b/src/i18n/locales/es/settings.json @@ -248,7 +248,7 @@ "auto": "Automática", "custom": "Personalizada", "mbpsInput": "Tasa de bits personalizada en Mbps", - "range": "1–200 Mbps" + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" }, "hevcHint": "HEVC (H.265) crea archivos más pequeños pero puede no reproducirse en reproductores antiguos o web. La vista previa de Recordly no cambia.", "hardwareUnavailable": "No se encontró ningún codificador de hardware utilizable. Elige CPU o Auto, o actualiza el controlador de tu GPU.", diff --git a/src/i18n/locales/fr/settings.json b/src/i18n/locales/fr/settings.json index 96120855f..4f2e64599 100644 --- a/src/i18n/locales/fr/settings.json +++ b/src/i18n/locales/fr/settings.json @@ -248,7 +248,7 @@ "auto": "Auto", "custom": "Personnalisé", "mbpsInput": "Débit binaire personnalisé en Mbit/s", - "range": "1–200 Mbit/s" + "range": "1–105 Mbit/s (H.264) · 1–70 Mbit/s (HEVC)" }, "hevcHint": "HEVC (H.265) crée des fichiers plus petits mais peut ne pas être lu sur les lecteurs anciens ou web. L'aperçu de Recordly reste inchangé.", "hardwareUnavailable": "Aucun encodeur matériel utilisable trouvé. Choisissez CPU ou Auto, ou mettez à jour le pilote de votre GPU.", diff --git a/src/i18n/locales/it/settings.json b/src/i18n/locales/it/settings.json index fb7a5c868..917611c55 100644 --- a/src/i18n/locales/it/settings.json +++ b/src/i18n/locales/it/settings.json @@ -248,7 +248,7 @@ "auto": "Auto", "custom": "Personalizzato", "mbpsInput": "Bitrate personalizzato in Mbps", - "range": "1–200 Mbps" + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" }, "hevcHint": "HEVC (H.265) produce file più piccoli ma potrebbe non essere riprodotto su lettori vecchi o web. L'anteprima di Recordly rimane invariata.", "hardwareUnavailable": "Nessun encoder hardware utilizzabile trovato. Scegli CPU o Auto, oppure aggiorna il driver della GPU.", diff --git a/src/i18n/locales/ko/settings.json b/src/i18n/locales/ko/settings.json index 46343237c..b816a8c12 100644 --- a/src/i18n/locales/ko/settings.json +++ b/src/i18n/locales/ko/settings.json @@ -268,7 +268,7 @@ "auto": "자동", "custom": "사용자 지정", "mbpsInput": "Mbps 단위 사용자 지정 비트레이트", - "range": "1–200 Mbps" + "range": "1–105 Mbps (H.264) · 1–70 Mbps (HEVC)" }, "hevcHint": "HEVC(H.265)는 파일을 더 작게 만들지만 이전 또는 웹 플레이어에서 재생되지 않을 수 있습니다. Recordly 미리보기 재생은 변경되지 않습니다.", "hardwareUnavailable": "사용 가능한 하드웨어 인코더를 찾을 수 없습니다. CPU 또는 자동을 선택하거나 GPU 드라이버를 업데이트하세요.", diff --git a/src/lib/exporter/exportBitrate.test.ts b/src/lib/exporter/exportBitrate.test.ts index 9301d4e0d..8e6126e7a 100644 --- a/src/lib/exporter/exportBitrate.test.ts +++ b/src/lib/exporter/exportBitrate.test.ts @@ -162,6 +162,8 @@ describe("export bitrate resolver", () => { it("custom mode converts Mbps to whole-number bps", () => { expect(customBitrateMbpsToBps(20)).toBe(20_000_000); expect(customBitrateMbpsToBps(12.5)).toBe(12_500_000); + expect(customBitrateMbpsToBps(200, "h264")).toBe(105_000_000); + expect(customBitrateMbpsToBps(200, "hevc")).toBe(70_000_000); }); it("custom mode clamps to the supported range and defaults non-finite input", () => { @@ -180,13 +182,26 @@ describe("export bitrate resolver", () => { resolveExportBitrate({ mode: "custom", customMbps: 500, + codec: "h264", width: 1920, height: 1080, frameRate: 30, quality: "source", encodingMode: "quality", }), - ).toBe(200_000_000); + ).toBe(105_000_000); + expect( + resolveExportBitrate({ + mode: "custom", + customMbps: 500, + codec: "hevc", + width: 1920, + height: 1080, + frameRate: 30, + quality: "source", + encodingMode: "quality", + }), + ).toBe(70_000_000); expect( resolveExportBitrate({ mode: "custom", @@ -231,6 +246,10 @@ describe("export bitrate resolver", () => { expect(clampCustomBitrateMbps(12.5)).toBe(12.5); expect(clampCustomBitrateMbps(0.5)).toBe(1); expect(clampCustomBitrateMbps(500)).toBe(200); + expect(clampCustomBitrateMbps(500, "h264")).toBe(105); + expect(clampCustomBitrateMbps(500, "hevc")).toBe(70); + expect(clampCustomBitrateMbps(12.5, "h264")).toBe(12.5); + expect(clampCustomBitrateMbps(70, "hevc")).toBe(70); expect(clampCustomBitrateMbps(NaN)).toBe(20); }); }); diff --git a/src/lib/exporter/exportBitrate.ts b/src/lib/exporter/exportBitrate.ts index 99bdfa583..9718f6b2c 100644 --- a/src/lib/exporter/exportBitrate.ts +++ b/src/lib/exporter/exportBitrate.ts @@ -1,11 +1,14 @@ import { EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_H264_MAX_MBPS, + EXPORT_BITRATE_HEVC_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS, EXPORT_BITRATE_MIN_MBPS, type ExportBitrateMode, type ExportEncodingMode, type ExportMp4FrameRate, type ExportQuality, + type ExportVideoCodec, } from "./types"; const MIN_MP4_BITRATE = 2_000_000; @@ -134,21 +137,32 @@ export function getMp4ExportBitrate(options: { return Math.max(MIN_MP4_BITRATE, cappedBitrate); } -export function clampCustomBitrateMbps(mbps: number): number { +function getCodecCustomBitrateCapMbps(codec: ExportVideoCodec | undefined): number { + switch (codec) { + case "h264": + return Math.min(EXPORT_BITRATE_H264_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS); + case "hevc": + return Math.min(EXPORT_BITRATE_HEVC_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS); + default: + return EXPORT_BITRATE_MAX_MBPS; + } +} + +export function clampCustomBitrateMbps(mbps: number, codec?: ExportVideoCodec): number { if (!Number.isFinite(mbps) || Number.isNaN(mbps)) { return EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS; } if (mbps < EXPORT_BITRATE_MIN_MBPS) { return EXPORT_BITRATE_MIN_MBPS; } - if (mbps > EXPORT_BITRATE_MAX_MBPS) { - return EXPORT_BITRATE_MAX_MBPS; + if (mbps > getCodecCustomBitrateCapMbps(codec)) { + return getCodecCustomBitrateCapMbps(codec); } return mbps; } -export function customBitrateMbpsToBps(mbps: number): number { - return Math.floor(clampCustomBitrateMbps(mbps) * 1_000_000); +export function customBitrateMbpsToBps(mbps: number, codec?: ExportVideoCodec): number { + return Math.floor(clampCustomBitrateMbps(mbps, codec) * 1_000_000); } export function resolveExportBitrate(options: { @@ -160,9 +174,10 @@ export function resolveExportBitrate(options: { quality: ExportQuality; encodingMode: ExportEncodingMode; useModernNativeStaticLayout?: boolean; + codec?: ExportVideoCodec; }): number { if (options.mode === "custom") { - return customBitrateMbpsToBps(options.customMbps); + return customBitrateMbpsToBps(options.customMbps, options.codec); } return getMp4ExportBitrate(options); } diff --git a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts index d630b13a5..aac003e50 100644 --- a/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts +++ b/src/lib/exporter/modernVideoExporter.nativeStaticLayout.test.ts @@ -24,6 +24,7 @@ function createExporter(overrides: Record = {}) { electronAPI: { nativeStaticLayoutExport: vi.fn(), nativeStaticLayoutExportCancel: vi.fn(), + discardExportedTemp: vi.fn(async () => ({ success: true })), }, }); @@ -207,6 +208,11 @@ describe("ModernVideoExporter native static-layout eligibility", () => { await expect( exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), ).resolves.toBeNull(); + // The produced temp video must not stay on disk for the session when the + // native result is rejected because it cannot satisfy the strict route. + expect(window.electronAPI.discardExportedTemp).toHaveBeenCalledWith( + "C:/Temp/hevc-cuda.mp4", + ); }); it("rejects the H.264-only Windows GPU route for HEVC", async () => { @@ -225,6 +231,11 @@ describe("ModernVideoExporter native static-layout eligibility", () => { await expect( exporter.tryExportNativeStaticLayout(videoInfo, { audioMode: "none" }, 60, 1_800), ).resolves.toBeNull(); + // The produced temp video must not stay on disk for the session when the + // native result is rejected because it cannot satisfy the requested codec. + expect(window.electronAPI.discardExportedTemp).toHaveBeenCalledWith( + "C:/Temp/hevc-windows-gpu.mp4", + ); }); it("allows native static-layout for H.264 source metadata", () => { diff --git a/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts index 74c633c2f..5e07e7c65 100644 --- a/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts +++ b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts @@ -39,6 +39,16 @@ const mocks = vi.hoisted(() => { frameRendererRenderOverlayFrame: vi.fn(async (timestampUs: number) => { framesRendered.push(timestampUs); }), + // Cursor-sprite capture defaults to unavailable so the baked-cursor + // full-canvas sidecar fallback is the default behavior. Dedicated + // cursor-sprite tests override these to exercise the sprite success path. + frameRendererStartCursorSpriteCapture: vi.fn(() => false), + frameRendererCaptureCursorSpriteFrame: vi.fn(() => ({ + captured: false, + unavailableReason: "cursor sprite unavailable (mock)", + })), + frameRendererFinishCursorSpriteCapture: vi.fn(() => null), + frameRendererCancelCursorSpriteCapture: vi.fn(() => {}), }; }); @@ -49,6 +59,10 @@ vi.mock("./modernFrameRenderer", () => ({ getCanvas: mocks.frameRendererGetCanvas, initialize: mocks.frameRendererInitialize, renderOverlayFrame: mocks.frameRendererRenderOverlayFrame, + startCursorSpriteCapture: mocks.frameRendererStartCursorSpriteCapture, + captureCursorSpriteFrame: mocks.frameRendererCaptureCursorSpriteFrame, + finishCursorSpriteCapture: mocks.frameRendererFinishCursorSpriteCapture, + cancelCursorSpriteCapture: mocks.frameRendererCancelCursorSpriteCapture, }; }), })); @@ -177,12 +191,16 @@ function createExporter(overrides: Record = {}) { videoInfo: DecodedVideoInfo, durationSec: number, totalFrames: number, + cursorExcluded?: boolean, + onPreparationProgress?: (renderProgress: number) => void, ) => Promise<{ overlayLayers: Array>; tiledOverlayLayers: Array>; rawFallbackReason: string | null; } | null>; nativeStaticLayoutOverlayFailure: { stage: string; message: string } | null; + nativeStaticLayoutSkipReason: string | null; + nativeStaticLayoutSkipReasons: string[]; hasNativeStaticLayoutOverlayContent: () => boolean; tryExportNativeStaticLayout: ( videoInfo: DecodedVideoInfo, @@ -221,6 +239,12 @@ describe("ModernVideoExporter native overlay preparation", () => { buffer.fill(value); }; vi.clearAllMocks(); + // Reset cursor-sprite mock implementations so the default (unavailable) + // fallback applies unless a test explicitly opts into the sprite path. + mocks.frameRendererStartCursorSpriteCapture.mockReset(); + mocks.frameRendererCaptureCursorSpriteFrame.mockReset(); + mocks.frameRendererFinishCursorSpriteCapture.mockReset(); + mocks.frameRendererCancelCursorSpriteCapture.mockReset(); vi.unstubAllGlobals(); }); @@ -254,7 +278,11 @@ describe("ModernVideoExporter native overlay preparation", () => { }); expect(result?.rawFallbackReason).toBeNull(); expect(result?.tiledOverlayLayers[0]?.staticTiles).toHaveLength(TILE_COUNT); - expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + // The cursor-sprite path is attempted first (atlas not actually owned) and + // falls back to the baked sidecar when sprite capture is unavailable, so + // the overlay renderer is initialized once for the sprite attempt and + // once for the baked full-canvas sidecar. + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(2); expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); @@ -266,7 +294,7 @@ describe("ModernVideoExporter native overlay preparation", () => { expect(lastTiledWrite[1]).toBe((TILE_COUNT - 1) * TILE_BYTE_SIZE); expect(lastTiledWrite[2]).toHaveLength(TILE_BYTE_SIZE); expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); - expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); expect(exporter.nativeStaticLayoutOverlayFailure).toBeNull(); }); @@ -290,7 +318,8 @@ describe("ModernVideoExporter native overlay preparation", () => { }); expect("effectiveFrameCount" in (result?.overlayLayers[0] ?? {})).toBe(false); expect(result?.rawFallbackReason).toBe("dense-frame-delta"); - expect(api.openExportStream).toHaveBeenCalledTimes(1); + // sprite + json export streams (sprite attempt) plus the baked rgba stream. + expect(api.openExportStream).toHaveBeenCalledTimes(3); expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); expect(api.writeExportStreamChunk).toHaveBeenCalledTimes(30); const lastWrite = api.writeExportStreamChunk.mock.calls[29] as [string, number, Uint8Array]; @@ -350,18 +379,29 @@ describe("ModernVideoExporter native overlay preparation", () => { message: expect.stringContaining("Overlay renderer is not initialized"), }); expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba", { abort: true }); - expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(1); + // One renderer for the sprite attempt, one for the baked sidecar. + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); }); it("records an overlay-stream-truncated failure stage when the sidecar byte count is short", async () => { vi.stubGlobal("VideoFrame", FakeVideoFrame); vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); const api = createWindowStub(); - api.closeExportStream.mockResolvedValueOnce({ - success: true, - tempPath: "C:/Temp/overlay.rgba", - bytesWritten: FRAME_BYTE_SIZE - 1, - }); + // Only the baked rgba stream finalize returns the truncated byte count; + // the cursor-sprite abort closes (sprite/json) return their defaults. + api.closeExportStream.mockImplementation( + async (streamId: string, options?: { abort?: boolean }) => { + const tempPath = `C:/Temp/overlay.${String(streamId).replace("overlay-", "")}`; + if (options?.abort) { + return { success: true, tempPath, bytesWritten: 0 }; + } + return { + success: true, + tempPath, + bytesWritten: streamId === "overlay-rgba" ? FRAME_BYTE_SIZE - 1 : 0, + }; + }, + ); const exporter = createExporter(); const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); @@ -496,7 +536,7 @@ describe("ModernVideoExporter native overlay preparation", () => { expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba"); expect(api.closeExportStream).toHaveBeenCalledWith("overlay-tiledrgba"); expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/overlay.rgba"); - expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); }); it("routes HEVC Hardware with overlay content to the native CUDA compositor with the tiled sidecar", async () => { @@ -518,7 +558,12 @@ describe("ModernVideoExporter native overlay preparation", () => { }, }); - const exporter = createExporter(); + // Cursor motion blur is a browser-rendered effect, so the cursor is baked + // into the transparent overlay sidecar (cursor-sidecar) rather than owned + // natively. This keeps the overlay-content path using existing preparation + // even when the deterministic no-browser-overlay fast lane is otherwise + // eligible. + const exporter = createExporter({ cursorMotionBlur: 1 }); const result = await exporter.tryExportNativeStaticLayout( videoInfo, { audioMode: "none" }, @@ -549,4 +594,845 @@ describe("ModernVideoExporter native overlay preparation", () => { encoderPreference: "hardware", }); }); + + it("returns no overlay sidecar when native CUDA owns the cursor and there are no browser-rendered pixels", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // cursorTelemetry + showCursor are set, but cursor render is excluded from + // the sidecar (native ownership) and there are no captions/annotations/ + // webcam/frame pixels, so the empty validated representation is correct. + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + expect(mocks.frameRendererDestroy).not.toHaveBeenCalled(); + }); + + it("still prepares the overlay sidecar when browser-rendered pixels coexist with native cursor ownership", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter({ frame: { enabled: true, width: 400, height: 300 } }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBeNull(); + }); + + it("coalesces and throttles preparation progress during tiled sidecar generation", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + createWindowStub(); + + const prepProgress: number[] = []; + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + false, + false, + (renderProgress) => prepProgress.push(renderProgress), + ); + + expect(result).not.toBeNull(); + // Every frame is identical, so all 30 frames would emit a raw per-frame + // update without coalescing. The bounded cadence must stay far below that. + expect(prepProgress.length).toBeGreaterThan(0); + expect(prepProgress.length).toBeLessThan(30); + for (const value of prepProgress) { + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThanOrEqual(100); + } + }); + + it("records a cancellation failure and aborts the overlay stream when cancelled mid-preparation", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + const exporter = createExporter(); + (exporter as unknown as { cancelled: boolean }).cancelled = true; + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutOverlayFailure).toMatchObject({ + stage: "overlay-preparation", + }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-rgba", { abort: true }); + // One renderer for the sprite attempt, one for the baked sidecar. + expect(mocks.frameRendererDestroy).toHaveBeenCalledTimes(2); + }); + + it("reports an initial preparing route progress that identifies the CUDA compositor first", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { + chunkCount: 1, + chunkDurationSec: 120, + chunkExecMs: 0, + chunks: [], + }, + }); + + const emitted: Array> = []; + const exporter = createExporter({ + onProgress: (progress: Record) => emitted.push(progress), + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + expect(emitted.length).toBeGreaterThan(0); + expect(emitted[0]).toMatchObject({ + phase: "preparing", + currentFrame: 0, + encoderName: "nvidia-cuda-compositor", + encodeBackend: "ffmpeg", + }); + }); + + it("discards the produced temp video when the native route cannot preserve zoom motion blur", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-blur-route.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Spatial zoom motion blur over overlay content requires the generalized + // CUDA compositor. The FFmpeg effectful overlay route cannot preserve it, + // so the successful native result is rejected; the produced temp video + // (potentially GBs for HEVC) must not be left on disk for the session. + const exporter = createExporter({ + exportEncoderPreference: "auto", + zoomMotionBlur: 0.35, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe( + "unsupported-motion-blur-on-overlay-route", + ); + expect(exporter.nativeStaticLayoutSkipReasons).toContain( + "unsupported-motion-blur-on-overlay-route", + ); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/hevc-blur-route.mp4"); + }); + + it("prepares a cursor-sprite overlay layer for a renderer-baked cursor on the CUDA route", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + const width = 32; + const height = 32; + const frameCount = 30; + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width, + height, + frameCount, + frames: new Uint8Array(width * height * 4 * frameCount), + positions: Array.from({ length: frameCount }, (_, index) => ({ + x: 10 + index, + y: 20, + })), + }); + + // Cursor motion blur disables native atlas ownership, so the cursor is a + // renderer-baked ROI sprite instead of a full transparent canvas sidecar. + const exporter = createExporter({ cursorMotionBlur: 1 }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + frameCount, + false, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + order: 1, + x: 0, + y: 0, + width, + height, + frameRate: 30, + durationSec: 1, + frameCount, + pixelFormat: "rgba", + }); + expect(layer.positions).toHaveLength(frameCount); + expect(mocks.frameRendererStartCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererCaptureCursorSpriteFrame).toHaveBeenCalledTimes(frameCount); + expect(mocks.frameRendererFinishCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererCancelCursorSpriteCapture).toHaveBeenCalledTimes(1); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "json" }); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-sprite"); + expect(api.closeExportStream).toHaveBeenCalledWith("overlay-json"); + }); + + it("uses the cursor-sprite path when the native atlas is eligible but not actually owned", async () => { + vi.stubGlobal("navigator", { platform: "Win32", userAgent: "node" }); + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // With the Win32 CUDA route the native atlas is *eligible*, but the atlas + // was not actually built/owned (cursorExcluded === false). The sprite path + // must run as the pixel-preserving fallback instead of forcing the + // expensive full-canvas tiled sidecar. Before the fix the atlas-eligibility + // gate blocked this and baked a full 4K sidecar per frame. + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("falls back to the baked cursor overlay sidecar when cursor-sprite capture is unavailable", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + // startCursorSpriteCapture defaults to false, so the cursor stays baked in + // the full transparent canvas sidecar (the preserved golden path). + + const exporter = createExporter({ cursorMotionBlur: 1 }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("rejects a non-CUDA native route that cannot compose the cursor sprite", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 1, y: 1 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, () => ({ x: 1, y: 1 })), + }); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "cuda-overlay", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + const exporter = createExporter({ cursorMotionBlur: 1, exportEncoderPreference: "auto" }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-cursor-sprite-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-cursor-sprite-route"); + }); + + it("uses the cursor-sprite ROI for a cursor-only H.264 CUDA export instead of baking a full 4K sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // H.264 CUDA-opt-in export with a cursor-only overlay (no browser pixels + // and no native atlas ownership). The generalized NVIDIA CUDA compositor + // consumes the cursor-sprite contract regardless of output codec, so the + // cheap ROI strip must be selected instead of baking the cursor into a full + // transparent 4K canvas per frame. Regression: the sprite path was gated on + // the HEVC-only canUseNativeGpuStaticLayout(), forcing H.264 CUDA cursor + // exports onto the ~1 minute full-canvas tiled sidecar this case reproduced. + const exporter = createExporter({ + exportVideoCodec: "h264", + exportEncoderPreference: "hardware", + }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("coalesces a long identical raw overlay run into bounded contiguous IPC writes", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + // 29 identical frames followed by one different frame produce a 29-frame + // identical middle run that must be coalesced into bounded contiguous IPC + // chunks instead of one writeExportStreamChunk call per frame. + frameSource.fill = (frameIndex: number, buffer: Uint8Array | Uint8ClampedArray) => { + buffer.fill(frameIndex >= 29 ? 0x11 : 0xaa); + }; + const api = createWindowStub(); + + const exporter = createExporter(); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30); + + // The dense full-frame delta on the last frame keeps this on the raw sidecar. + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBe("dense-frame-delta"); + const rgbaWrites = api.writeExportStreamChunk.mock.calls.filter( + (call: [string, number, Uint8Array]) => call[0] === "overlay-rgba", + ) as Array<[string, number, Uint8Array]>; + // The 29 identical frames are coalesced; strictly fewer IPC calls than the + // 30 frames they represent. + expect(rgbaWrites.length).toBeLessThan(30); + // The coalesced batches must cover every overlay frame byte exactly once, + // preserving offsets and content (no dropped or duplicated pixels). + const totalBytes = rgbaWrites.reduce((sum, call) => sum + call[2].byteLength, 0); + expect(totalBytes).toBe(30 * FRAME_BYTE_SIZE); + let cursor = 0; + for (const [, offset, chunk] of rgbaWrites) { + expect(offset % FRAME_BYTE_SIZE).toBe(0); + expect(chunk.byteLength % FRAME_BYTE_SIZE).toBe(0); + expect(offset).toBe(cursor); + cursor += chunk.byteLength; + } + // The final (different) frame is present at the correct byte offset with the + // correct value. + expect(rgbaWrites[rgbaWrites.length - 1]?.[1]).toBe(29 * FRAME_BYTE_SIZE); + expect(rgbaWrites[rgbaWrites.length - 1]?.[2][0]).toBe(0x11); + }); + + it("uses the cursor-sprite ROI for the exact HEVC Hardware cursor-only CUDA case instead of a tiled sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // The user's reported case: HEVC Hardware with a cursor-only overlay (zoom + // is native, so it is not browser pixel content) on the CUDA route. The + // cursor-sprite ROI strip must be selected instead of baking a full + // transparent 4K canvas sidecar (which surfaces as tiledOverlayLayers: 1). + const exporter = createExporter({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, false); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("emits one-shot preparation stage diagnostics (overlay + IPC handoff) with codec/preference/route and elapsedMs", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { + chunkCount: 1, + chunkDurationSec: 120, + chunkExecMs: 0, + chunks: [], + }, + }); + + const infoMessages: Array<{ message: string; payload: Record }> = []; + const infoSpy = vi + .spyOn(console, "info") + .mockImplementation((first?: unknown, second?: unknown, third?: unknown) => { + const text = String(second ?? ""); + if (text.includes("Native static layout preparation stage")) { + infoMessages.push({ + message: text, + payload: (third ?? {}) as Record, + }); + } + }); + + const exporter = createExporter(); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const overlayStage = infoMessages.find((entry) => entry.payload.stage === "overlay"); + const ipcStage = infoMessages.find((entry) => entry.payload.stage === "ipc-handoff"); + expect(overlayStage).toBeDefined(); + expect(ipcStage).toBeDefined(); + expect(overlayStage?.payload).toMatchObject({ + exportVideoCodec: "hevc", + exportEncoderPreference: "hardware", + route: "nvidia-cuda-compositor", + }); + expect(typeof overlayStage?.payload.elapsedMs).toBe("number"); + expect((overlayStage?.payload.elapsedMs as number) ?? 0).toBeGreaterThanOrEqual(0); + expect(ipcStage?.payload).toMatchObject({ + route: "nvidia-cuda-compositor", + success: true, + }); + infoSpy.mockRestore(); + }); + + it("makes the tiled route reason explicit when browser-rendered pixels coexist with the cursor", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + createWindowStub(); + + const infoPayloads: Array> = []; + const infoSpy = vi + .spyOn(console, "info") + .mockImplementation((message?: unknown, payload?: unknown) => { + if (String(message ?? "").includes("Cursor-sprite overlay path skipped")) { + infoPayloads.push((payload ?? {}) as Record); + } + }); + + // Native cursor ownership is active (Win32 CUDA route, no cursor effects) + // but a browser-rendered frame forces the baked full-canvas sidecar, which + // resolves to the tiled representation. This is necessary, not a sprite + // failure, and must be surfaced explicitly. + const exporter = createExporter({ frame: { enabled: true, width: 400, height: 300 } }); + const result = await exporter.prepareNativeStaticLayoutOverlay(videoInfo, 1, 30, true); + + expect(result).not.toBeNull(); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(infoPayloads).toHaveLength(1); + expect(infoPayloads[0]).toMatchObject({ + reason: "browser-overlay-pixels", + bakedSidecarRequired: true, + browserPixelSources: ["frame"], + }); + infoSpy.mockRestore(); + }); + + it("returns no overlay sidecar when the CUDA compositor owns a webcam-only overlay (webcam+zoom fast path)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // HEVC Hardware CUDA route with the webcam as the ONLY browser-rendered + // pixel (zoom is native, cursor is atlas-owned). The webcam is excluded + // from the renderer sidecar and the CUDA compositor draws it from + // webcamInputPath, so the sidecar is provably empty and must not render or + // read back a full 4K canvas per frame. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + true, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(0); + expect(result?.tiledOverlayLayers).toHaveLength(0); + expect(result?.rawFallbackReason).toBeNull(); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + expect(mocks.frameRendererDestroy).not.toHaveBeenCalled(); + }); + + it("still renders the baked webcam sidecar when captions coexist with the webcam (mixed overlay fallback)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // Captions are browser-rendered pixels, so the webcam cannot be owned + // natively: the existing baked full-canvas sidecar path must run unchanged + // (webcam stays in the sidecar; webcamInputPath is never sent alongside + // baked pixels, which prevents double-draw). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + autoCaptions: [{ startMs: 0, endMs: 1000, text: "Hi", lang: "en" }], + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + false, + ); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "tiledrgba" }); + expect(result?.tiledOverlayLayers).toHaveLength(1); + expect(result?.rawFallbackReason).toBeNull(); + }); + + it("keeps the webcam baked when a webcam shadow would be lost on the CUDA route", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + + // The CUDA compositor webcam overlay has no shadow support, so a shadowed + // webcam must stay on the baked sidecar path even when it is the only + // browser pixel. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4", shadow: 0.5 }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + true, + false, + ); + + expect(result).not.toBeNull(); + expect(mocks.frameRendererInitialize).toHaveBeenCalledTimes(1); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "rgba" }); + }); + + it("uses the cursor-sprite ROI for a webcam-native export and never bakes the webcam into the sidecar", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // Webcam-only browser pixels with a renderer-baked cursor: the webcam is + // excluded from the sidecar (native CUDA ownership) and the cursor is + // captured as the cheap ROI sprite, so no full 4K canvas is rendered per + // frame and the webcam is never double-drawn. + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.prepareNativeStaticLayoutOverlay( + videoInfo, + 1, + 30, + false, + true, + ); + + expect(result).not.toBeNull(); + expect(result?.overlayLayers).toHaveLength(1); + expect(result?.tiledOverlayLayers).toHaveLength(0); + const layer = result?.overlayLayers[0] as Record; + expect(layer).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + width: 32, + height: 32, + frameCount: 30, + }); + expect(api.openExportStream).toHaveBeenCalledWith({ extension: "sprite" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "rgba" }); + expect(api.openExportStream).not.toHaveBeenCalledWith({ extension: "tiledrgba" }); + }); + + it("passes webcamInputPath and webcamNativeOwned for a webcam+zoom HEVC Hardware CUDA export", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Webcam is the only browser pixel and the cursor is disabled, so the + // deterministic fast lane selects an empty sidecar while the CUDA + // compositor owns the webcam natively: no renderer init, no per-frame + // canvas readback, and the webcam must reach the CUDA wrapper. + const exporter = createExporter({ + showCursor: false, + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].overlayLayers).toBeUndefined(); + expect(exportCall[0].tiledOverlayLayers).toBeUndefined(); + expect(exportCall[0].webcamInputPath).toBe("C:/webcam.mp4"); + expect(exportCall[0].webcamNativeOwned).toBe(true); + expect(mocks.frameRendererInitialize).not.toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).not.toHaveBeenCalled(); + expect(api.openExportStream).not.toHaveBeenCalled(); + }); + + it("passes webcamInputPath alongside a cursor-sprite overlay without double-drawing the webcam", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + + // Webcam native ownership + a renderer-baked cursor: the sidecar holds + // only the cursor-sprite ROI and the webcam still reaches the CUDA + // compositor, so the webcam is drawn exactly once (never also baked). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].overlayLayers).toHaveLength(1); + expect((exportCall[0].overlayLayers as Array>)[0]).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + }); + expect(exportCall[0].webcamInputPath).toBe("C:/webcam.mp4"); + expect(exportCall[0].webcamNativeOwned).toBe(true); + }); + + it("keeps webcamNativeOwned undefined and webcamInputPath null when captions coexist (baked path)", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-static.mp4", + videoCodec: "hevc", + encoderPreference: "hardware", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Captions are browser-rendered pixels, so the webcam stays baked in the + // sidecar and webcamInputPath must NOT be sent: sending it would make the + // CUDA compositor draw the webcam a second time (double-draw). + const exporter = createExporter({ + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + autoCaptions: [{ startMs: 0, endMs: 1000, text: "Hi", lang: "en" }], + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].webcamInputPath).toBeNull(); + expect(exportCall[0].webcamNativeOwned).toBeUndefined(); + // The baked sidecar resolves to the tiled representation for static + // content: overlayLayers stays undefined and tiledOverlayLayers carries + // the sidecar pixels (cursor + captions + webcam baked together). + expect(exportCall[0].overlayLayers).toBeUndefined(); + expect(Array.isArray(exportCall[0].tiledOverlayLayers)).toBe(true); + expect((exportCall[0].tiledOverlayLayers as unknown[]).length).toBeGreaterThan(0); + expect(mocks.frameRendererInitialize).toHaveBeenCalled(); + expect(mocks.frameRendererRenderOverlayFrame).toHaveBeenCalledTimes(30); + }); + + it("keeps the webcam baked for HEVC Auto (non-strict) even when it is the only browser pixel", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/hevc-auto.mp4", + videoCodec: "hevc", + encoderPreference: "auto", + route: "nvidia-cuda-compositor", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // HEVC Auto is not the strict CUDA-only route: a fallback could still + // render the webcam from the baked sidecar, so the webcam stays baked and + // is never excluded from the sidecar. + const exporter = createExporter({ + showCursor: false, + exportEncoderPreference: "auto", + webcam: { enabled: true, sourcePath: "C:/webcam.mp4" }, + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toMatchObject({ success: true }); + const exportCall = api.nativeStaticLayoutExport.mock.calls[0] as [Record]; + expect(exportCall[0].webcamNativeOwned).toBeUndefined(); + expect(exportCall[0].webcamInputPath).toBeNull(); + expect(api.openExportStream).toHaveBeenCalled(); + expect(mocks.frameRendererInitialize).toHaveBeenCalled(); + }); }); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 74b1d2650..9e3f04023 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -52,6 +52,7 @@ import { DEFAULT_WALLPAPER_RELATIVE_PATH, isVideoWallpaperSource, } from "@/lib/wallpapers"; +import { formatLogTs } from "../log"; import { AudioProcessor, isAacAudioEncodingSupported } from "./audioEncoder"; import { normalizeLightningRuntimePlatform, @@ -86,6 +87,8 @@ import { VideoMuxer } from "./muxer"; import { captureCanvasFrameForNativeExport } from "./nativeFrameCapture"; import { roundNativeStaticLayoutContentSize } from "./nativeStaticLayoutGeometry"; import type { + NativeCursorSpriteOverlayLayer, + NativeCursorSpritePosition, NativeStaticLayoutOverlayLayer, NativeTiledOverlayFrameDelta, NativeTiledOverlayLayerDescriptor, @@ -95,11 +98,13 @@ import type { } from "./nativeStaticLayoutOverlays"; import { areNativeStaticLayoutOverlayFramesEqual, + clampNativeCursorSpritePosition, getNativeStaticLayoutOverlayFrameByteSize, getNativeTiledOverlayTileColumns, getNativeTiledOverlayTileCount, getNativeTiledOverlayTileIndex, getNativeTiledOverlayTileRows, + NATIVE_CURSOR_SPRITE_LAYER_KIND, NATIVE_TILED_OVERLAY_MAX_CHANGED_TILE_FRACTION, NATIVE_TILED_OVERLAY_MAX_PAYLOAD_BYTES_FRACTION, NATIVE_TILED_OVERLAY_PIXEL_FORMAT, @@ -108,6 +113,7 @@ import { resolveNativeTiledOverlayRawFallbackReason, sortNativeStaticLayoutOverlayLayers, sortNativeTiledOverlayLayers, + validateNativeCursorSpriteOverlayLayer, } from "./nativeStaticLayoutOverlays"; import { buildNativeStaticLayoutCursorTelemetry } from "./nativeStaticLayoutTelemetry"; import { resolveSourceAudioFallbackPaths } from "./sourceAudioFallback"; @@ -203,12 +209,24 @@ interface VideoExporterConfig extends ExportConfig { * in `electron/ipc/export/native-video.ts` already validates, sorts, and * composites both lists. */ + +type NativeStaticLayoutOverlayLayerUnion = + | NativeStaticLayoutOverlayLayer + | NativeCursorSpriteOverlayLayer; + type NativeStaticLayoutOverlayPreparationResult = { - overlayLayers: NativeStaticLayoutOverlayLayer[]; + overlayLayers: NativeStaticLayoutOverlayLayerUnion[]; tiledOverlayLayers: NativeTiledOverlayLayerDescriptor[]; rawFallbackReason: NativeTiledOverlayRawFallbackReason | null; }; +/** Discriminates a cursor-sprite layer from a fixed-position rgba layer. */ +function isCursorSpriteOverlayLayer( + layer: NativeStaticLayoutOverlayLayerUnion, +): layer is NativeCursorSpriteOverlayLayer { + return (layer as { kind?: string }).kind === NATIVE_CURSOR_SPRITE_LAYER_KIND; +} + type NativeAudioPlan = | { audioMode: "none"; @@ -360,6 +378,14 @@ const STATIC_LAYOUT_CHUNK_DURATION_SEC = 120; const MISSING_NATIVE_WALLPAPER_FALLBACK_COLOR = "#ffffff"; const NATIVE_STATIC_LAYOUT_MAX_EXTRACTING_PROGRESS = 95; const NATIVE_STATIC_LAYOUT_FRAME_COMPLETE_PROGRESS = 96; +const NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS = 300; + +// Upper bound in bytes for a single coalesced overlay sidecar IPC chunk. +// Pixel-identical consecutive frames in the raw sidecar are coalesced into one +// contiguous chunk (instead of one writeExportStreamChunk call per frame) to +// remove per-frame IPC round-trips for static overlay stretches, while capping +// the chunk so a long identical run never buffers the whole 4K sidecar at once. +const NATIVE_RAW_OVERLAY_RUN_BATCH_MAX_BYTES = 48 * 1024 * 1024; const HEVC_NATIVE_STATIC_LAYOUT_ROUTES = new Set([ "cuda-overlay", "cuda-scale-cpu-pad", @@ -401,6 +427,63 @@ export function shouldRejectNativeStaticLayoutResultForEffectPreservation(option return requiresCudaCompositor && options.route !== "nvidia-cuda-compositor"; } +/** + * Detailed skip reason for the deterministic no-browser-overlay fast lane. + */ +export type NativeStaticLayoutFastLaneSkipReason = + | "not-native-cuda-route" + | "browser-overlay-pixels-present" + | "cursor-sidecar-required" + | "edited-audio-render-required" + | "native-source-not-authoritative"; + +export type NativeStaticLayoutFastLaneEligibility = { + eligible: boolean; + skipReasons: NativeStaticLayoutFastLaneSkipReason[]; +}; + +/** + * Deterministic no-browser-overlay fast lane for native CUDA static-layout + * export. When every visual input the browser would otherwise render is already + * authoritative natively (source video is a local file, no captions/annotations/ + * webcam/frame pixels, the cursor is either disabled or owned by the native CUDA + * compositor, and there is no edited-audio render), the export can skip renderer + * initialization, per-frame canvas capture, overlay sidecar creation, and cursor + * atlas generation and start the native export as early as safely allowed. + * + * The predicate is explicit and returns detailed skip reasons so callers can + * prove (and log) exactly why the fast lane was or was not selected. It never + * bypasses source validation/security, required audio muxing, timeline/zoom/ + * temporal native plans, cancellation/cleanup/progress settlement, or strict HEVC + * Hardware CUDA-only failure behavior. + */ +export function getNativeStaticLayoutFastLaneEligibility(options: { + canUseNativeGpuStaticLayout: boolean; + hasBrowserOverlayPixels: boolean; + cursorDisabled: boolean; + cursorNativeOwnershipActive: boolean; + requiresEditedAudioRender: boolean; + hasAuthoritativeNativeSource: boolean; +}): NativeStaticLayoutFastLaneEligibility { + const skipReasons: NativeStaticLayoutFastLaneSkipReason[] = []; + if (!options.canUseNativeGpuStaticLayout) { + skipReasons.push("not-native-cuda-route"); + } + if (options.hasBrowserOverlayPixels) { + skipReasons.push("browser-overlay-pixels-present"); + } + if (!options.cursorDisabled && !options.cursorNativeOwnershipActive) { + skipReasons.push("cursor-sidecar-required"); + } + if (options.requiresEditedAudioRender) { + skipReasons.push("edited-audio-render-required"); + } + if (!options.hasAuthoritativeNativeSource) { + skipReasons.push("native-source-not-authoritative"); + } + return { eligible: skipReasons.length === 0, skipReasons }; +} + export class ModernVideoExporter { private static readonly NATIVE_ENCODER_QUEUE_LIMIT = 64; private static readonly NATIVE_WRITE_BATCH_MAX_CHUNKS = 12; @@ -482,6 +565,7 @@ export class ModernVideoExporter { private lastProgressSampleTimeMs = 0; private lastProgressSampleFrame = 0; private displayedRenderFps = 0; + private lastPreparingTotalFrames: number | null = null; constructor(config: VideoExporterConfig) { this.config = config; @@ -532,6 +616,7 @@ export class ModernVideoExporter { // instead of falling through to the renderer raw/WebCodecs path. if (this.requiresStrictNativeCudaRoute() && !this.canUseNativeGpuStaticLayout()) { console.error( + formatLogTs(), "[VideoExporter] Strict HEVC Hardware policy: CUDA compositor not eligible; refusing renderer raw fallback", { exportVideoCodec: this.config.exportVideoCodec, @@ -629,7 +714,7 @@ export class ModernVideoExporter { frameRate: this.config.frameRate, encodingMode: this.config.encodingMode, }); - console.log("[VideoExporter] Native static-layout decision", { + console.log(formatLogTs(), "[VideoExporter] Native static-layout decision", { exportVideoCodec: this.config.exportVideoCodec ?? "h264", exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", experimentalNativeExport: this.config.experimentalNativeExport === true, @@ -643,6 +728,14 @@ export class ModernVideoExporter { zoomMotionBlur: this.config.zoomMotionBlur ?? 0, zoomTemporalMotionBlur: this.config.zoomTemporalMotionBlur ?? 0, hasOverlayContent: this.hasNativeStaticLayoutOverlayContent(), + hasBrowserOverlayPixels: this.hasNativeStaticLayoutBrowserOverlayPixels(), + showCursor: this.config.showCursor === true, + cursorTelemetrySamples: this.config.cursorTelemetry?.length ?? 0, + cursorMotionBlur: this.config.cursorMotionBlur ?? 0, + cursorSway: this.config.cursorSway ?? 0, + cursorAtlasOwnershipEligible: this.canUseNativeCursorAtlasOwnership(), + webcamOnlyBrowserPixels: this.hasNativeStaticLayoutWebcamOnlyBrowserPixels(), + webcamNativeOwnershipEligible: this.canUseNativeWebcamOwnership(), }); this.maxNativeWriteInFlight = useNativeEncoder ? Math.max( @@ -751,6 +844,7 @@ export class ModernVideoExporter { // failure). Never fall back to the renderer raw frame path, Breeze, // WebGPU, or CPU; hard-fail with the first skip reason. console.error( + formatLogTs(), "[VideoExporter] Strict HEVC Hardware policy: CUDA static-layout route did not render; refusing raw renderer fallback", { skipReason: this.nativeStaticLayoutSkipReason, @@ -774,6 +868,7 @@ export class ModernVideoExporter { // surfaced here so a raw hevc_nvenc session can always be traced // back to its cause. console.warn( + formatLogTs(), "[VideoExporter] Native static-layout route did not render; starting raw renderer frame path", { exportVideoCodec: this.config.exportVideoCodec ?? "h264", @@ -2551,6 +2646,54 @@ export class ModernVideoExporter { if (clickEffect !== undefined && clickEffect !== "none") { return false; } + if (this.hasNativeStaticLayoutExtensionCursorVisuals()) { + return false; + } + // Only the generalized NVIDIA CUDA compositor draws the atlas on top of + // the overlay sidecars; the FFmpeg overlay route and the D3D11 helper + // cannot, so native ownership requires the CUDA-opt-in Windows route. + return ( + this.getRuntimePlatform() === "win32" && + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true + ); + } + + /** + * Whether the generalized NVIDIA CUDA compositor is an eligible consumer of + * the native `cursor-sprite` overlay contract. + * + * The cursor-sprite contract captures only the small cursor ROI strip + * instead of baking the cursor into a full transparent 4K canvas per frame. + * It is consumed solely by the generalized NVIDIA CUDA compositor, which is + * independent of the output codec: native-video.ts runs the same CUDA + * compositor for H.264 and HEVC overlay exports whenever the user opts into + * the CUDA route. This predicate therefore gates on the CUDA route, not on + * the (HEVC-only) canUseNativeGpuStaticLayout(). Gating the cheap ROI path + * on the codec wrongly forced H.264 CUDA exports with a cursor-only overlay + * to bake the full-canvas sidecar frame-by-frame (~1 min for 192 frames) + * instead of capturing the tiny cursor ROI. + * + * CPU encoder preference never reaches the CUDA compositor (it is the + * software-encoder route), so it must never attempt a sprite here. + */ + private canUseNativeCursorSpriteContract(): boolean { + if (this.config.exportEncoderPreference === "cpu") { + return false; + } + return ( + this.config.experimentalNativeExport === true && + this.config.experimentalNvidiaCudaExport === true + ); + } + + /** + * Whether extension cursor visuals / render hooks are active. Extension + * hooks draw into the full composite canvas outside the cursor container, so + * any path that captures only the cursor container (cursor-sprite ROI) would + * silently drop them. The baked full-canvas sidecar is required instead. + */ + private hasNativeStaticLayoutExtensionCursorVisuals(): boolean { const extensionHookPhases = [ "background", "post-video", @@ -2560,19 +2703,9 @@ export class ModernVideoExporter { "post-annotations", "final", ] as const; - if ( + return ( extensionHost.hasCursorEffects() || extensionHookPhases.some((phase) => extensionHost.hasRenderHooks(phase)) - ) { - return false; - } - // Only the generalized NVIDIA CUDA compositor draws the atlas on top of - // the overlay sidecars; the FFmpeg overlay route and the D3D11 helper - // cannot, so native ownership requires the CUDA-opt-in Windows route. - return ( - this.getRuntimePlatform() === "win32" && - this.config.experimentalNativeExport === true && - this.config.experimentalNvidiaCudaExport === true ); } @@ -2586,6 +2719,65 @@ export class ModernVideoExporter { ); } + // Browser-rendered overlay pixels (everything the renderer draws into the + // transparent sidecar). When the native CUDA compositor owns the cursor atlas + // and none of these are present, the sidecar would be entirely transparent, + // so it can be skipped entirely without rendering/capturing a canvas per frame. + // When the webcam is owned natively by the CUDA compositor (webcamNativeOwned) + // the renderer must NOT bake it into the sidecar, so it is excluded from the + // browser-pixel check exactly like an atlas-owned cursor. + private hasNativeStaticLayoutBrowserOverlayPixels(webcamNativeOwned = false): boolean { + return Boolean( + (this.config.annotationRegions?.length ?? 0) > 0 || + (this.config.autoCaptions?.length ?? 0) > 0 || + Boolean(this.config.frame) || + (Boolean(this.config.webcam?.enabled) && !webcamNativeOwned), + ); + } + + /** + * Whether the webcam is the ONLY browser-rendered overlay pixel source and is + * fully representable by the generalized NVIDIA CUDA compositor's native + * webcam overlay contract. + * + * The CUDA compositor consumes the same resolved webcam geometry the renderer + * would bake (left/top/size/radius/mirror/time-offset via the native-video.ts + * webcam args), so a webcam-only export needs no renderer sidecar at all. + * Mixed browser content (captions, annotations, frame visuals) or extension + * render hooks keep the existing baked sidecar path, and a configured webcam + * shadow is not representable in the CUDA wrapper today, so a shadowed webcam + * must stay baked to preserve the golden visual. + */ + private hasNativeStaticLayoutWebcamOnlyBrowserPixels(): boolean { + const webcamOverlay = this.getNativeStaticLayoutWebcamOverlay(); + return ( + this.config.webcam?.enabled === true && + webcamOverlay !== null && + (webcamOverlay.shadowIntensity ?? 0) <= 0 && + (this.config.annotationRegions?.length ?? 0) === 0 && + (this.config.autoCaptions?.length ?? 0) === 0 && + !this.config.frame && + !this.hasNativeStaticLayoutExtensionCursorVisuals() + ); + } + + /** + * Whether the generalized NVIDIA CUDA compositor owns the webcam overlay + * natively for this export. + * + * Safe only on the strict HEVC Hardware CUDA route: that route guarantees the + * CUDA compositor runs (any fallback hard-fails with noCpuFallback:true), so + * excluding the webcam from the renderer sidecar can never silently drop it on + * an FFmpeg/D3D11 fallback that cannot draw a native webcam. HEVC Auto and + * H.264 keep the existing baked-webcam sidecar path unchanged. + */ + private canUseNativeWebcamOwnership(): boolean { + if (!this.requiresStrictNativeCudaRoute() || !this.canUseNativeGpuStaticLayout()) { + return false; + } + return this.hasNativeStaticLayoutWebcamOnlyBrowserPixels(); + } + private hasUnsupportedNativeStaticLayoutOverlayContent(): string | null { if (this.config.annotationRegions?.some((annotation) => annotation.type === "blur")) { return "unsupported-blur-annotation-overlay"; @@ -2593,9 +2785,34 @@ export class ModernVideoExporter { return null; } + private getNativeStaticLayoutFastLaneEligibility( + audioPlan: NativeAudioPlan, + cursorAtlasOwnedByNative: boolean, + webcamNativeOwned: boolean, + ): NativeStaticLayoutFastLaneEligibility { + const cursorDisabled = + this.config.showCursor !== true || (this.config.cursorTelemetry?.length ?? 0) === 0; + // Actual ownership, not eligibility: the empty sidecar fast lane is only + // safe when the cursor is disabled or the CUDA compositor will genuinely + // draw it from a successfully built atlas. An eligible-but-unbuilt atlas + // must not silently drop the cursor, so it keeps the sidecar preparation + // (cursor-sprite ROI or baked full-canvas) and never selects the fast lane. + const cursorNativeOwnershipActive = cursorAtlasOwnedByNative; + return getNativeStaticLayoutFastLaneEligibility({ + canUseNativeGpuStaticLayout: this.canUseNativeGpuStaticLayout(), + hasBrowserOverlayPixels: + this.hasNativeStaticLayoutBrowserOverlayPixels(webcamNativeOwned), + cursorDisabled, + cursorNativeOwnershipActive, + requiresEditedAudioRender: audioPlan.audioMode === "edited-track", + hasAuthoritativeNativeSource: Boolean(this.getNativeVideoSourcePath()), + }); + } + private createNativeStaticLayoutOverlayRenderer( videoInfo: DecodedVideoInfo, excludeCursorOverlay = false, + excludeWebcamOverlay = false, ) { return new ModernFrameRenderer({ width: this.config.width, @@ -2619,8 +2836,8 @@ export class ModernVideoExporter { borderRadius: this.config.borderRadius, padding: this.config.padding, cropRegion: this.config.cropRegion, - webcam: this.config.webcam, - webcamUrl: this.config.webcamUrl, + webcam: excludeWebcamOverlay ? undefined : this.config.webcam, + webcamUrl: excludeWebcamOverlay ? null : this.config.webcamUrl, videoWidth: videoInfo.width, videoHeight: videoInfo.height, annotationRegions: this.config.annotationRegions, @@ -2682,11 +2899,280 @@ export class ModernVideoExporter { } } + /** + * Whether the cursor should be captured as a cursor-sprite ROI strip instead + * of being baked into a full transparent RGBA canvas sidecar. + * + * A cursor-sprite is only usable on the generalized NVIDIA CUDA compositor + * (the sole consumer of the native `cursor-sprite` contract) and only when + * the cursor is the entire overlay (no browser pixels) and is NOT actually + * owned by the native atlas (cursorExcluded === cursorAtlasOwnedByNative). + * When the atlas is eligible but was not successfully built, the sprite path + * is the pixel-preserving fallback: it renders the same Pixi cursor into the + * ROI instead of the expensive full-canvas tiled sidecar. Browser-only + * cursor effects (motion blur/sway/click) also use the sprite. Extension + * cursor visuals keep the baked full-canvas sidecar because extension hooks + * draw outside the cursor container (the sprite would drop them). When the + * sprite cannot be used the baked-cursor full-canvas sidecar path runs + * unchanged (the preserved golden path). + */ + private shouldUseNativeStaticLayoutCursorSprite( + cursorExcluded: boolean, + webcamExcluded = false, + ): boolean { + return ( + !cursorExcluded && + this.canUseNativeCursorSpriteContract() && + this.config.showCursor === true && + (this.config.cursorTelemetry?.length ?? 0) > 0 && + !this.hasNativeStaticLayoutExtensionCursorVisuals() && + !this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded) + ); + } + + /** + * Captures the cursor ROI as a fixed packed RGBA sprite strip plus per-frame + * top-left positions and returns a validated native `cursor-sprite` overlay + * layer. Returns null (recording an overlay failure) when the cursor-sprite + * contract cannot be prepared, in which case the caller falls back to the + * existing baked-cursor full-canvas sidecar path. + */ + private async prepareNativeStaticLayoutCursorSprite( + videoInfo: DecodedVideoInfo, + durationSec: number, + totalFrames: number, + webcamExcluded = false, + onPreparationProgress?: (renderProgress: number) => void, + ): Promise { + const api = typeof window === "undefined" ? null : window.electronAPI; + if ( + !api?.openExportStream || + !api.writeExportStreamChunk || + !api.closeExportStream || + !api.discardExportedTemp + ) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-api-unavailable", + "Cursor-sprite export stream IPC is not available", + ); + return null; + } + const renderer = this.createNativeStaticLayoutOverlayRenderer( + videoInfo, + false, + webcamExcluded, + ); + let spriteStreamId: string | null = null; + let positionsStreamId: string | null = null; + try { + const spriteStream = await api.openExportStream({ extension: "sprite" }); + if (!spriteStream.success || !spriteStream.streamId || !spriteStream.tempPath) { + this.recordNativeStaticLayoutOverlayFailure( + "open-cursor-sprite-stream", + spriteStream.error ?? "Cursor-sprite export stream could not be opened", + ); + return null; + } + spriteStreamId = spriteStream.streamId; + + const positionsStream = await api.openExportStream({ extension: "json" }); + if ( + !positionsStream.success || + !positionsStream.streamId || + !positionsStream.tempPath + ) { + this.recordNativeStaticLayoutOverlayFailure( + "open-cursor-positions-stream", + positionsStream.error ?? + "Cursor-sprite positions export stream could not be opened", + ); + return null; + } + positionsStreamId = positionsStream.streamId; + + await renderer.initialize(); + const started = renderer.startCursorSpriteCapture(); + if (!started) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-init", + "Cursor-sprite capture could not be initialized (no overlay renderer)", + ); + return null; + } + + let lastPreparationProgressMs = 0; + for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) { + if (this.cancelled) { + throw new Error("Export cancelled"); + } + if (onPreparationProgress) { + const nowMs = this.getNowMs(); + if ( + nowMs - lastPreparationProgressMs >= + NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS || + frameIndex === totalFrames - 1 + ) { + lastPreparationProgressMs = nowMs; + onPreparationProgress( + totalFrames > 0 ? (frameIndex / totalFrames) * 100 : 0, + ); + } + } + const timestampUs = Math.round((frameIndex * 1_000_000) / this.config.frameRate); + try { + // The cursor-sprite path captures only the cursor ROI, so skip the + // full 4K canvas render that the baked full-canvas sidecar needs. + // All cursor state updates (sway spring, motion-blur velocity, + // click rings, zoom transform) still run; only the expensive + // full-canvas rasterization is skipped. + await renderer.renderOverlayFrame(timestampUs, timestampUs, timestampUs, true); + } catch (error) { + throw new Error( + `overlay-renderer-frame: ${error instanceof Error ? error.message : String(error)}`, + ); + } + const capture = renderer.captureCursorSpriteFrame(); + if (!capture.captured) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-frame", + capture.unavailableReason ?? "Cursor-sprite frame could not be captured", + ); + return null; + } + } + + const strip = renderer.finishCursorSpriteCapture(); + if (!strip || strip.frameCount === 0) { + this.recordNativeStaticLayoutOverlayFailure( + "cursor-sprite-finish", + "Cursor-sprite capture produced no frames", + ); + return null; + } + + const spriteWrite = await api.writeExportStreamChunk(spriteStreamId, 0, strip.frames); + if (!spriteWrite.success) { + throw new Error( + `cursor-sprite-stream-write: ${spriteWrite.error ?? "Failed to write cursor-sprite strip"}`, + ); + } + const clampedPositions: NativeCursorSpritePosition[] = strip.positions.map((position) => + clampNativeCursorSpritePosition( + position, + strip.width, + strip.height, + this.config.width, + this.config.height, + ), + ); + const positionsBytes = new TextEncoder().encode(JSON.stringify(clampedPositions)); + const positionsWrite = await api.writeExportStreamChunk( + positionsStreamId, + 0, + positionsBytes, + ); + if (!positionsWrite.success) { + throw new Error( + `cursor-positions-stream-write: ${positionsWrite.error ?? "Failed to write cursor-sprite positions"}`, + ); + } + + const spriteClosed = await api.closeExportStream(spriteStreamId); + spriteStreamId = null; + if (!spriteClosed.success || !spriteClosed.tempPath) { + throw new Error( + `cursor-sprite-stream-close: ${spriteClosed.error ?? "Cursor-sprite stream did not finalize"}`, + ); + } + const positionsClosed = await api.closeExportStream(positionsStreamId); + positionsStreamId = null; + if (!positionsClosed.success || !positionsClosed.tempPath) { + throw new Error( + `cursor-positions-stream-close: ${positionsClosed.error ?? "Cursor-sprite positions stream did not finalize"}`, + ); + } + + const layer: NativeCursorSpriteOverlayLayer = { + id: "cursor-sprite", + order: 1, + kind: NATIVE_CURSOR_SPRITE_LAYER_KIND, + path: spriteClosed.tempPath, + positionsPath: positionsClosed.tempPath, + x: 0, + y: 0, + width: strip.width, + height: strip.height, + frameRate: this.config.frameRate, + durationSec, + frameCount: strip.frameCount, + positions: clampedPositions, + pixelFormat: "rgba", + }; + const validationError = validateNativeCursorSpriteOverlayLayer(layer, { + outputWidth: this.config.width, + outputHeight: this.config.height, + durationSec, + frameRate: this.config.frameRate, + }); + if (validationError) { + throw new Error(`cursor-sprite-layer-invalid: ${validationError}`); + } + console.info("[VideoExporter] Native static layout cursor-sprite selected", { + route: "nvidia-cuda-compositor", + cursorStyle: this.config.cursorStyle ?? "tahoe", + spriteWidth: strip.width, + spriteHeight: strip.height, + frameCount: strip.frameCount, + }); + return layer; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + const stage = message.startsWith("cursor-") + ? message.split(":", 1)[0] + : "cursor-sprite-preparation"; + this.recordNativeStaticLayoutOverlayFailure(stage, message); + console.warn( + "[VideoExporter] Cursor-sprite preparation failed; falling back to baked cursor overlay sidecar", + { stage, message, totalFrames, cancelled: this.cancelled }, + ); + return null; + } finally { + // Abort any export streams still open on both the thrown-error path and + // the early-return-null paths (a finalized stream already nulls its id). + if (spriteStreamId) { + try { + await api.closeExportStream(spriteStreamId, { abort: true }); + } catch { + // Best-effort cleanup. + } + } + if (positionsStreamId) { + try { + await api.closeExportStream(positionsStreamId, { abort: true }); + } catch { + // Best-effort cleanup. + } + } + try { + renderer.cancelCursorSpriteCapture(); + } catch { + // Best-effort cleanup. + } + try { + renderer.destroy(); + } catch { + // Cleanup is best-effort after a failed cursor-sprite attempt. + } + } + } + private async prepareNativeStaticLayoutOverlay( videoInfo: DecodedVideoInfo, durationSec: number, totalFrames: number, cursorExcluded = false, + webcamExcluded = false, + onPreparationProgress?: (renderProgress: number) => void, ): Promise { this.nativeStaticLayoutOverlayFailure = null; if (!this.hasNativeStaticLayoutOverlayContent()) { @@ -2704,6 +3190,99 @@ export class ModernVideoExporter { ); return null; } + // Safe empty-work fast path: the native CUDA compositor owns the cursor + // atlas and there are no browser-rendered overlay pixels (captions, + // annotations, webcam, frame, or other). Rendering/capturing a full + // transparent canvas for every output frame would be pure waste, so return + // the validated empty overlay representation instead of a sidecar. Zoom and + // temporal motion-blur effects are preserved natively on the GPU and are + // unaffected by omitting an empty sidecar. + if (cursorExcluded && !this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded)) { + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + } + // Cursor-sprite path: when the cursor is the only overlay content (no + // browser pixels) and it cannot be owned by the native atlas, capture the + // cursor ROI as a packed RGBA strip + per-frame positions instead of + // writing a full transparent canvas sidecar for every output frame. Only + // the generalized NVIDIA CUDA compositor consumes the cursor-sprite + // contract. When the sprite cannot be prepared the existing baked-cursor + // full-canvas sidecar path below runs unchanged (the preserved golden + // path) and carries a clear diagnostic note. + const cursorSpriteEligible = this.shouldUseNativeStaticLayoutCursorSprite( + cursorExcluded, + webcamExcluded, + ); + const cursorSpriteLayer = cursorSpriteEligible + ? await this.prepareNativeStaticLayoutCursorSprite( + videoInfo, + durationSec, + totalFrames, + webcamExcluded, + onPreparationProgress, + ) + : null; + if (cursorSpriteLayer) { + return { + overlayLayers: sortNativeStaticLayoutOverlayLayers([cursorSpriteLayer]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + } + // Surface why the cursor-sprite path was not taken. When the path was + // eligible but preparation failed, prepareNativeStaticLayoutCursorSprite + // already logged the stage/message; only the eligibility miss needs an + // explicit note here (the baked sidecar is the preserved golden path). + if (!cursorSpriteEligible) { + const hasBrowserPixels = this.hasNativeStaticLayoutBrowserOverlayPixels(webcamExcluded); + const browserPixelSources: string[] = []; + if ((this.config.annotationRegions?.length ?? 0) > 0) { + browserPixelSources.push("annotations"); + } + if ((this.config.autoCaptions?.length ?? 0) > 0) { + browserPixelSources.push("captions"); + } + if (this.config.frame) { + browserPixelSources.push("frame"); + } + if (this.config.webcam?.enabled === true && !webcamExcluded) { + browserPixelSources.push("webcam"); + } + const spriteAvailable = + this.canUseNativeCursorSpriteContract() && + this.config.showCursor === true && + (this.config.cursorTelemetry?.length ?? 0) > 0; + const reason = hasBrowserPixels + ? "browser-overlay-pixels" + : this.hasNativeStaticLayoutExtensionCursorVisuals() + ? "extension-cursor-visuals" + : !spriteAvailable + ? "cursor-sprite-contract-unavailable" + : "cursor-excluded-by-native-atlas"; + console.info("[VideoExporter] Cursor-sprite overlay path skipped", { + route: "nvidia-cuda-compositor", + reason, + bakedSidecarRequired: reason !== "cursor-excluded-by-native-atlas", + browserPixelSources, + hasExtensionCursorVisuals: this.hasNativeStaticLayoutExtensionCursorVisuals(), + cursorExcluded, + showCursor: this.config.showCursor === true, + cursorTelemetrySamples: this.config.cursorTelemetry?.length ?? 0, + cursorAtlasOwnershipEligible: this.canUseNativeCursorAtlasOwnership(), + hasBrowserOverlayPixels: hasBrowserPixels, + annotationRegions: this.config.annotationRegions?.length ?? 0, + autoCaptions: this.config.autoCaptions?.length ?? 0, + frame: Boolean(this.config.frame), + webcamEnabled: this.config.webcam?.enabled === true, + }); + } + // Falling back to the baked-cursor full-canvas sidecar; clear any + // cursor-sprite preparation failure so a successful sidecar is not + // misreported as an overlay failure. + this.nativeStaticLayoutOverlayFailure = null; const api = typeof window === "undefined" ? null : window.electronAPI; if ( !api?.openExportStream || @@ -2738,7 +3317,11 @@ export class ModernVideoExporter { return null; } - const renderer = this.createNativeStaticLayoutOverlayRenderer(videoInfo, cursorExcluded); + const renderer = this.createNativeStaticLayoutOverlayRenderer( + videoInfo, + cursorExcluded, + webcamExcluded, + ); const frameByteSize = getNativeStaticLayoutOverlayFrameByteSize( this.config.width, this.config.height, @@ -2770,15 +3353,16 @@ export class ModernVideoExporter { return null; } const activeRawStreamId: string = rawStreamId; - const writeRawOverlayFrame = async ( + const writeRawOverlayChunk = async ( frameIndex: number, - frame: Uint8Array, + frameCount: number, + chunk: Uint8Array, ): Promise => { try { const result = await api.writeExportStreamChunk( activeRawStreamId, frameIndex * frameByteSize, - frame, + chunk, ); if (!result.success) { throw new Error(result.error ?? "Failed to write native overlay frame"); @@ -2788,28 +3372,63 @@ export class ModernVideoExporter { `overlay-stream-write: ${error instanceof Error ? error.message : String(error)}`, ); } - rawWrittenFrameCount += 1; + rawWrittenFrameCount += frameCount; }; const flushRawIdenticalRun = async (untilFrameIndex: number): Promise => { - if (runFrame === null) { + if (runFrame === null || untilFrameIndex <= runStartFrameIndex) { return; } - for ( - let frameIndex = runStartFrameIndex; - frameIndex < untilFrameIndex; - frameIndex += 1 - ) { - await writeRawOverlayFrame(frameIndex, runFrame); + const runLength = untilFrameIndex - runStartFrameIndex; + const framesPerBatch = Math.max( + 1, + Math.floor(NATIVE_RAW_OVERLAY_RUN_BATCH_MAX_BYTES / frameByteSize), + ); + let batchStartFrameIndex = runStartFrameIndex; + while (batchStartFrameIndex < untilFrameIndex) { + const batchFrameCount = Math.min( + runLength - (batchStartFrameIndex - runStartFrameIndex), + framesPerBatch, + ); + if (batchFrameCount === 1) { + await writeRawOverlayChunk(batchStartFrameIndex, 1, runFrame); + } else { + const batchBytes = batchFrameCount * frameByteSize; + const batch = new Uint8Array(batchBytes); + for (let i = 0; i < batchFrameCount; i += 1) { + batch.set(runFrame, i * frameByteSize); + } + await writeRawOverlayChunk(batchStartFrameIndex, batchFrameCount, batch); + } + batchStartFrameIndex += batchFrameCount; } }; let rawTempPath: string | null = null; + let lastPreparationProgressMs = 0; try { await renderer.initialize(); for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) { if (this.cancelled) { throw new Error("Export cancelled"); } + // Coalesced preparation heartbeat: report at most once per throttle + // interval (plus a final report on the last frame) so the UI stays + // responsive during long sidecar generation without one React update + // per frame. This is preparation progress only; render FPS is never + // faked here (currentFrame stays 0 in the preparing phase). + if (onPreparationProgress) { + const nowMs = this.getNowMs(); + if ( + nowMs - lastPreparationProgressMs >= + NATIVE_OVERLAY_PREPARATION_PROGRESS_INTERVAL_MS || + frameIndex === totalFrames - 1 + ) { + lastPreparationProgressMs = nowMs; + onPreparationProgress( + totalFrames > 0 ? (frameIndex / totalFrames) * 100 : 0, + ); + } + } const timestampUs = Math.round((frameIndex * 1_000_000) / this.config.frameRate); try { await renderer.renderOverlayFrame(timestampUs); @@ -2919,7 +3538,7 @@ export class ModernVideoExporter { } if (runFrame !== null) { - await writeRawOverlayFrame(runStartFrameIndex, runFrame); + await writeRawOverlayChunk(runStartFrameIndex, 1, runFrame); } let rawClosed: Awaited>; @@ -3096,6 +3715,22 @@ export class ModernVideoExporter { } } + private logNativeStaticLayoutPreparationStage( + stage: string, + startedAtMs: number, + extra: Record = {}, + ): void { + const elapsedMs = Math.round(this.getNowMs() - startedAtMs); + console.info(formatLogTs(), "[VideoExporter] Native static layout preparation stage", { + stage, + elapsedMs, + exportVideoCodec: this.config.exportVideoCodec ?? "h264", + exportEncoderPreference: this.config.exportEncoderPreference ?? "auto", + route: this.canUseNativeGpuStaticLayout() ? "nvidia-cuda-compositor" : "static-layout", + ...extra, + }); + } + private recordNativeStaticLayoutOverlayFailure(stage: string, message: string): void { this.nativeStaticLayoutOverlayFailure = { stage, message }; } @@ -3117,7 +3752,7 @@ export class ModernVideoExporter { if (skipReason) { this.nativeStaticLayoutSkipReason = skipReason; this.nativeStaticLayoutSkipReasons = skipReasons; - console.info("[VideoExporter] Native static layout skipped", { + console.info(formatLogTs(), "[VideoExporter] Native static layout skipped", { route: "native-static-layout", fallbackRoute: "breeze-stream-or-raw-frame", reason: skipReason, @@ -3141,8 +3776,31 @@ export class ModernVideoExporter { return null; } + // Emit the initial "preparing" signal before the potentially long + // audio/background/cursor/overlay preparation begins, and identify the + // NVIDIA CUDA compositor as the selected route when it is eligible so the + // first progress never shows a stale WebGPU/Breeze/libx264 backend during + // CUDA preparation. + this.encodeBackend = "ffmpeg"; + this.encoderName = this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : this.config.experimentalNativeExport === true && this.getRuntimePlatform() === "win32" + ? "windows-native-compositor" + : "static-layout-h264-nvenc"; + this.exportStartTimeMs = this.getNowMs(); + this.lastProgressSampleTimeMs = this.exportStartTimeMs; + this.lastProgressSampleFrame = 0; + this.reportProgress(0, totalFrames, "preparing"); + + let preparationStageStartedAt = this.getNowMs(); const sourcePath = this.getNativeVideoSourcePath(); const audioOptions = await this.getNativeStaticLayoutAudioOptions(audioPlan, totalFrames); + this.logNativeStaticLayoutPreparationStage("audio", preparationStageStartedAt, { + audioMode: audioPlan.audioMode, + editedTrackStrategy: + audioPlan.audioMode === "edited-track" ? audioPlan.strategy : undefined, + }); + preparationStageStartedAt = this.getNowMs(); if (!sourcePath || !audioOptions) { this.nativeStaticLayoutSkipReason = !sourcePath ? "missing-source-path" @@ -3151,6 +3809,12 @@ export class ModernVideoExporter { return null; } const background = await this.resolveNativeStaticLayoutBackground(); + this.logNativeStaticLayoutPreparationStage("background", preparationStageStartedAt, { + backgroundColor: background?.backgroundColor ?? null, + hasBackgroundImage: Boolean(background?.backgroundImagePath), + backgroundSkipReason: this.nativeStaticLayoutBackgroundSkipReason ?? null, + }); + preparationStageStartedAt = this.getNowMs(); if (!background) { this.nativeStaticLayoutSkipReason = this.nativeStaticLayoutBackgroundSkipReason ?? "unsupported-background"; @@ -3198,6 +3862,7 @@ export class ModernVideoExporter { ? Math.min(1, Math.max(0, this.config.shadowIntensity)) : 0; const webcamOverlay = this.getNativeStaticLayoutWebcamOverlay(); + const webcamNativeOwned = this.canUseNativeWebcamOwnership(); const cursorTelemetry = this.getNativeStaticLayoutCursorTelemetry(); const zoomTelemetry = this.getNativeStaticLayoutZoomTelemetry( layout, @@ -3234,6 +3899,12 @@ export class ModernVideoExporter { }, ) : null; + this.logNativeStaticLayoutPreparationStage("cursor-atlas", preparationStageStartedAt, { + wantsNativeCursorOwnership, + cursorAtlasBuilt: Boolean(cursorAtlas), + atlasEntries: cursorAtlas?.entries.length ?? 0, + }); + preparationStageStartedAt = this.getNowMs(); const cursorAtlasOwnedByNative = wantsNativeCursorOwnership && Boolean(cursorAtlas); if (cursorAtlasOwnedByNative) { console.info("[VideoExporter] Native cursor atlas owns the overlay cursor", { @@ -3262,14 +3933,69 @@ export class ModernVideoExporter { await this.cleanupNativeStaticLayoutBackground(background); return null; } - const overlayPreparation = await this.prepareNativeStaticLayoutOverlay( - videoInfo, - effectiveDuration, - totalFrames, + const fastLaneEligibility = this.getNativeStaticLayoutFastLaneEligibility( + audioPlan, cursorAtlasOwnedByNative, + webcamNativeOwned, ); + const useFastLane = fastLaneEligibility.eligible; + let overlayPreparation: NativeStaticLayoutOverlayPreparationResult | null = null; + if (useFastLane) { + // Deterministic no-browser-overlay fast lane: with no captions, + // annotations, or frame pixels (and the webcam owned natively by the CUDA + // compositor when enabled) and the cursor either disabled or owned + // natively by the CUDA compositor, the sidecar is provably empty, so + // skip renderer init, per-frame canvas capture, and overlay sidecar + // creation and start the native export as early as safely allowed. + overlayPreparation = { + overlayLayers: sortNativeStaticLayoutOverlayLayers([]), + tiledOverlayLayers: sortNativeTiledOverlayLayers([]), + rawFallbackReason: null, + }; + console.info(formatLogTs(), "[VideoExporter] Native static layout fast lane selected", { + route: "nvidia-cuda-compositor", + skipReasons: fastLaneEligibility.skipReasons, + cursorDisabled: + this.config.showCursor !== true || + (this.config.cursorTelemetry?.length ?? 0) === 0, + cursorNativeOwnershipActive: Boolean(cursorAtlasOwnedByNative), + webcamNativeOwned: Boolean(webcamNativeOwned), + audioMode: audioPlan.audioMode, + preparedOverlayLayers: overlayPreparation.overlayLayers.length, + preparedTiledOverlayLayers: overlayPreparation.tiledOverlayLayers.length, + }); + } else { + overlayPreparation = await this.prepareNativeStaticLayoutOverlay( + videoInfo, + effectiveDuration, + totalFrames, + cursorAtlasOwnedByNative, + webcamNativeOwned, + (renderProgress) => + this.reportProgress(0, totalFrames, "preparing", renderProgress), + ); + } const overlayLayers = overlayPreparation?.overlayLayers ?? []; const tiledOverlayLayers = overlayPreparation?.tiledOverlayLayers ?? []; + this.logNativeStaticLayoutPreparationStage("overlay", preparationStageStartedAt, { + mode: useFastLane + ? "fast-lane" + : !overlayPreparation + ? "failed" + : tiledOverlayLayers.length > 0 + ? "tiled-sidecar" + : overlayLayers.some(isCursorSpriteOverlayLayer) + ? "cursor-sprite" + : overlayLayers.length > 0 + ? "raw-sidecar" + : "empty", + overlayLayerCount: overlayLayers.length, + tiledOverlayLayerCount: tiledOverlayLayers.length, + rawFallbackReason: overlayPreparation?.rawFallbackReason ?? null, + overlayFailure: this.nativeStaticLayoutOverlayFailure, + webcamNativeOwned: Boolean(webcamNativeOwned), + }); + preparationStageStartedAt = this.getNowMs(); if (needsOverlayLayers && !overlayPreparation) { this.nativeStaticLayoutSkipReason = "native-overlay-preparation-failed"; const overlayFailure = this.nativeStaticLayoutOverlayFailure; @@ -3281,6 +4007,7 @@ export class ModernVideoExporter { ] : [this.nativeStaticLayoutSkipReason]; console.warn( + formatLogTs(), "[VideoExporter] Native static layout skipped: overlay preparation failed", { reason: this.nativeStaticLayoutSkipReason, @@ -3320,8 +4047,9 @@ export class ModernVideoExporter { typeof navigator !== "undefined" ? normalizeLightningRuntimePlatform(navigator.userAgent) : "unknown"; - this.encoderName = - this.config.experimentalNativeExport === true && runtimePlatform === "win32" + this.encoderName = this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : this.config.experimentalNativeExport === true && runtimePlatform === "win32" ? "windows-native-compositor" : "static-layout-h264-nvenc"; this.reportProgress(0, totalFrames, "preparing"); @@ -3410,6 +4138,7 @@ export class ModernVideoExporter { // Preparation-inclusive estimate; never presented as measured encode speed. this.nativeStaticLayoutFpsSource = "estimated"; console.warn( + formatLogTs(), "[VideoExporter] Native encode FPS not reported yet; using preparation-inclusive estimate", { backend: progress.backend, estimatedFps }, ); @@ -3456,8 +4185,15 @@ export class ModernVideoExporter { backgroundBlurPx: Math.max(0, (this.config.backgroundBlur ?? 0) * 3), borderRadius, shadowIntensity, - webcamInputPath: - overlayLayers.length || tiledOverlayLayers.length + // When the webcam is native-owned the renderer excluded it from the + // overlay sidecar, so webcamInputPath must reach the CUDA compositor + // even when a cursor-sprite (or baked-cursor) overlay layer is present. + // Mixed baked content (captions/annotations/frame) never sets + // webcamNativeOwned, so the existing baked-webcam contract (no + // webcamInputPath alongside sidecar pixels) is preserved. + webcamInputPath: webcamNativeOwned + ? (webcamOverlay?.inputPath ?? null) + : overlayLayers.length || tiledOverlayLayers.length ? null : (webcamOverlay?.inputPath ?? null), webcamLeft: webcamOverlay?.left, @@ -3467,6 +4203,10 @@ export class ModernVideoExporter { webcamShadowIntensity: webcamOverlay?.shadowIntensity, webcamMirror: webcamOverlay?.mirror, webcamTimeOffsetMs: webcamOverlay?.timeOffsetMs, + // True only when the CUDA compositor owns the webcam: the overlay + // sidecar excluded webcam pixels and the native webcam overlay must + // draw them (never double-render a baked webcam). + webcamNativeOwned: webcamNativeOwned || undefined, cursorTelemetry, cursorSize: this.getNativeStaticLayoutCursorSize(contentWidth), cursorAtlasPngDataUrl: cursorAtlas?.dataUrl ?? null, @@ -3490,9 +4230,18 @@ export class ModernVideoExporter { outputDurationSec: effectiveDuration, }, }; + const ipcHandoffStartedAt = this.getNowMs(); const result = await window.electronAPI.nativeStaticLayoutExport(nativeStaticLayoutOptions); - + this.logNativeStaticLayoutPreparationStage("ipc-handoff", ipcHandoffStartedAt, { + route: result.route ?? null, + success: result.success, + requestedVideoCodec, + requestedEncoderPreference, + requestedRoute: this.canUseNativeGpuStaticLayout() + ? "nvidia-cuda-compositor" + : "static-layout", + }); if (this.cancelled) { return { success: false, @@ -3506,9 +4255,13 @@ export class ModernVideoExporter { typeof result.error === "string" && result.error.trim() ? result.error.trim() : "unknown-native-static-layout-export-error"; - console.warn("[VideoExporter] Native static layout export unavailable", { - error: exportError, - }); + console.warn( + formatLogTs(), + "[VideoExporter] Native static layout export unavailable", + { + error: exportError, + }, + ); // Surface the real IPC/helper failure instead of a generic // "route unavailable" when strict HEVC Hardware later refuses the // renderer raw fallback. The strict error carries this detail so CUDA @@ -3534,6 +4287,13 @@ export class ModernVideoExporter { ); this.nativeStaticLayoutSkipReason = routeSkipReason; this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs for + // HEVC); discard it before falling back so it is not left on disk for + // the whole session. Best-effort: cleanup must never override the + // intended skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); restoreEncoderState(); return null; } @@ -3561,10 +4321,52 @@ export class ModernVideoExporter { ); this.nativeStaticLayoutSkipReason = routeSkipReason; this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs for + // HEVC); discard it before falling back so it is not left on disk for + // the whole session. Best-effort: cleanup must never override the + // intended skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); restoreEncoderState(); return null; } - console.info("[VideoExporter] Native static layout selected", { + // A cursor-sprite layer is only composited by the generalized NVIDIA + // CUDA compositor. If the actual route is anything else (FFmpeg effectful + // overlay or D3D11 helper) it would silently drop the cursor, so reject + // and let the renderer raw-frame fallback keep it. + const hasCursorSpriteLayer = overlayLayers.some((layer) => + isCursorSpriteOverlayLayer(layer), + ); + if (hasCursorSpriteLayer && result.route !== "nvidia-cuda-compositor") { + const routeSkipReason = "unsupported-cursor-sprite-route"; + console.warn( + "[VideoExporter] Rejecting native static-layout result that cannot compose the cursor sprite", + { route: result.route, layerCount: overlayLayers.length }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + restoreEncoderState(); + return null; + } + // A native-owned webcam is only drawn by the generalized NVIDIA CUDA + // compositor (the sidecar excluded webcam pixels). Any other route would + // silently drop the webcam, so reject and let the renderer raw-frame + // fallback keep it instead. Strict HEVC Hardware already refuses non-CUDA + // routes; this guard is the explicit observable invariant for webcam + // ownership on every codec/preference combination. + if (webcamNativeOwned && result.route !== "nvidia-cuda-compositor") { + const routeSkipReason = "unsupported-native-webcam-route"; + console.warn( + "[VideoExporter] Rejecting native static-layout result that cannot draw the native-owned webcam", + { route: result.route, webcamNativeOwned }, + ); + this.nativeStaticLayoutSkipReason = routeSkipReason; + this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + restoreEncoderState(); + return null; + } + console.info(formatLogTs(), "[VideoExporter] Native static layout selected", { route: result.route, encoderName: result.encoderName, exportVideoCodec: requestedVideoCodec, @@ -3573,6 +4375,7 @@ export class ModernVideoExporter { experimentalNativeExport: this.config.experimentalNativeExport === true, experimentalNvidiaCudaExport: this.config.experimentalNvidiaCudaExport === true, hasOverlayLayers: this.hasNativeStaticLayoutOverlayContent(), + webcamNativeOwned: Boolean(webcamNativeOwned), temporalBlurSamples: getTemporalMotionBlurConfig(this.config.zoomTemporalMotionBlur, { sampleCount: this.config.zoomMotionBlurSampleCount, @@ -3644,7 +4447,11 @@ export class ModernVideoExporter { }; } - console.warn("[VideoExporter] Native static layout export failed; falling back", error); + console.warn( + formatLogTs(), + "[VideoExporter] Native static layout export failed; falling back", + error, + ); const failureMessage = error instanceof Error ? error.message : String(error); this.lastNativeExportError = failureMessage; this.nativeStaticLayoutSkipReason = "native-static-runtime-failed"; @@ -4466,6 +5273,22 @@ export class ModernVideoExporter { renderProgress?: number, audioProgress?: number, ) { + // Suppress repeated identical "preparing" start signals (0 frames, no render + // or audio progress) during a single export so the renderer/UI is not + // spammed with identical progress resets. The first signal per total frame + // count is still delivered and progress semantics are unchanged. + const isIdenticalPreparingSignal = + phase === "preparing" && + currentFrame === 0 && + renderProgress === undefined && + audioProgress === undefined; + if (isIdenticalPreparingSignal && this.lastPreparingTotalFrames === totalFrames) { + return; + } + if (isIdenticalPreparingSignal) { + this.lastPreparingTotalFrames = totalFrames; + } + const nowMs = this.getNowMs(); const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001); const averageRenderFps = currentFrame / elapsedSeconds; @@ -5020,6 +5843,7 @@ export class ModernVideoExporter { this.lastProgressSampleTimeMs = 0; this.lastProgressSampleFrame = 0; this.displayedRenderFps = 0; + this.lastPreparingTotalFrames = null; this.nativeWritePromises = new Set(); this.nativeRawWritePromises = new Set(); this.nativeRawBackpressure = null; diff --git a/src/lib/exporter/types.ts b/src/lib/exporter/types.ts index fd5453e40..7ce1812ad 100644 --- a/src/lib/exporter/types.ts +++ b/src/lib/exporter/types.ts @@ -236,6 +236,8 @@ export interface ExportSettings { export const EXPORT_BITRATE_MIN_MBPS = 1; export const EXPORT_BITRATE_MAX_MBPS = 200; +export const EXPORT_BITRATE_H264_MAX_MBPS = 105; +export const EXPORT_BITRATE_HEVC_MAX_MBPS = 70; export const EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS = 20; export const MP4_FRAME_RATES: readonly ExportMp4FrameRate[] = [24, 30, 60] as const; From 994974eb31ff50bc264db4f8a66ef2651f0c0623 Mon Sep 17 00:00:00 2001 From: nmzpy <246990748+nmzpy@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:24:43 +0300 Subject: [PATCH 10/11] fix(export): address CodeRabbit round-2 review - fail with unsupported-temporal-motion-blur below the CUDA minimum sample count instead of warning - attach manifest kind/order to overlay layers so classification and z-order survive - cache the native --help capability probe per export - discard produced temp video on cursor-sprite and webcam route rejection - reset the preparing progress watermark after non-preparing progress - reuse codec bitrate caps in the export settings and re-clamp on codec switch - cap HEVC auto bitrate at 70 Mbps - propagate noCpuFallback through HEVC Hardware post-route failures - hoist requestSettings computation; align native-video test assertion; restore cursor-sprite test defaults --- electron/ipc/export/native-video.test.ts | 16 +- .../nativeStaticLayoutRoutePlan.test.ts | 32 ++ .../ipc/export/nativeStaticLayoutRoutePlan.ts | 11 +- electron/ipc/register/export.ts | 33 +- .../overlayManifest.mjs | 141 ++++++++ .../overlayManifest.test.mjs | 311 ++++++++++++++++++ .../run-mp4-pipeline.mjs | 32 +- .../video-editor/ExportSettingsMenu.tsx | 35 +- src/lib/exporter/exportBitrate.test.ts | 30 ++ src/lib/exporter/exportBitrate.ts | 10 +- src/lib/exporter/index.ts | 29 +- ...rnVideoExporter.overlayPreparation.test.ts | 43 ++- .../modernVideoExporter.progressDedup.test.ts | 150 +++++++++ ...modernVideoExporter.routeRejection.test.ts | 277 ++++++++++++++++ src/lib/exporter/modernVideoExporter.ts | 21 ++ 15 files changed, 1120 insertions(+), 51 deletions(-) create mode 100644 src/lib/exporter/modernVideoExporter.progressDedup.test.ts create mode 100644 src/lib/exporter/modernVideoExporter.routeRejection.test.ts diff --git a/electron/ipc/export/native-video.test.ts b/electron/ipc/export/native-video.test.ts index 234a94bb2..8465b7d2a 100644 --- a/electron/ipc/export/native-video.test.ts +++ b/electron/ipc/export/native-video.test.ts @@ -429,10 +429,18 @@ describe("native static-layout source probe cache", () => { encodingMode: "speed", }), ).toBe(false); - const current = baseCurrent(); - current.requestedCodec = "hevc"; - current.encoderPreference = "hardware"; - expect(canReuseNativeStaticLayoutSourceProbe(baseEntry(), current)).toBe(true); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + requestedCodec: "h264", + }), + ).toBe(false); + expect( + canReuseNativeStaticLayoutSourceProbe(entry, { + ...baseCurrent(), + encoderPreference: "cpu", + }), + ).toBe(false); }); }); diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts index 265f4b0e3..3f9bfd4cb 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.test.ts @@ -177,6 +177,38 @@ describe("planNativeStaticLayoutRoutes", () => { ); }); + it("keeps the strict no-CPU-fallback flag on HEVC Hardware post-route failures", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "hardware", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + // Route is selected (CUDA available), but if it fails after planning the + // plan must still forbid CPU/rawvideo/Breeze fallback. + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.noCpuFallback).toBe(true); + expect(plan.fallbackRoute).toBe("hard-fail"); + expect(plan.fallbackReason).toBe("hevc-hardware-route-failed"); + }); + + it("keeps the HEVC Auto selected plan non-strict for post-route failures", () => { + const plan = planNativeStaticLayoutRoutes({ + videoCodec: "hevc", + encoderPreference: "auto", + cuda: cudaProbe, + d3d11: d3d11Probe, + source, + }); + + expect(plan.selectedRoute).toBe("nvidia-cuda-compositor"); + expect(plan.noCpuFallback).toBe(false); + expect(plan.fallbackRoute).toBeNull(); + expect(plan.fallbackReason).toBeNull(); + }); + it("hard-fails when HEVC CUDA is unavailable and Hardware is strict", () => { const plan = planNativeStaticLayoutRoutes({ videoCodec: "hevc", diff --git a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts index f7a19600c..25c0d57d0 100644 --- a/electron/ipc/export/nativeStaticLayoutRoutePlan.ts +++ b/electron/ipc/export/nativeStaticLayoutRoutePlan.ts @@ -153,13 +153,18 @@ export function planNativeStaticLayoutRoutes(options: { status: "rejected", reasons: ["hevc-requires-nvidia-cuda-compositor"], }); + // Strict HEVC Hardware policy survives route selection: if the selected + // CUDA compositor fails after planning, the export must still hard-fail + // instead of falling back to renderer raw frames, Breeze, or CPU. HEVC + // Auto keeps the non-strict contract (rawvideo fallback is allowed). + const strictHevcHardware = encoderPreference === "hardware"; return { videoCodec, encoderPreference, selectedRoute: "nvidia-cuda-compositor", - fallbackRoute: null, - fallbackReason: null, - noCpuFallback: false, + fallbackRoute: strictHevcHardware ? "hard-fail" : null, + fallbackReason: strictHevcHardware ? "hevc-hardware-route-failed" : null, + noCpuFallback: strictHevcHardware, decisions, cuda, d3d11, diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index d7e70f1ed..4df40e7c2 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -328,6 +328,19 @@ export function registerExportHandlers() { const videoCodec = options.videoCodec ?? "h264"; const encoderPreference = options.encoderPreference ?? "auto"; let sessionId = ""; + // Build the request summary once, before the try, so the same string is + // reused for the start and failure logs (formatNativeExportRequestSettings + // is a pure string formatter over already-normalized locals; it has no + // side effects that must stay inside the try). + const requestSettings = formatNativeExportRequestSettings({ + videoCodec, + encoderPreference, + inputMode, + encodingMode: options.encodingMode, + width: options.width, + height: options.height, + frameRate: options.frameRate, + }); try { if (options.width % 2 !== 0 || options.height % 2 !== 0) { throw new Error("Native export requires even output dimensions"); @@ -341,15 +354,6 @@ export function registerExportHandlers() { // request line below is emitted before encoder resolution so a mismatch // between the requested prewarm (e.g. hevc/hardware) and the encoder the // export actually started with is immediately visible in one place. - const requestSettings = formatNativeExportRequestSettings({ - videoCodec, - encoderPreference, - inputMode, - encodingMode: options.encodingMode, - width: options.width, - height: options.height, - frameRate: options.frameRate, - }); console.log( formatLogTs(), `[native-export] Start request session=${sessionId} ${requestSettings}`, @@ -500,18 +504,9 @@ export function registerExportHandlers() { encoderName, }; } catch (error) { - const failedRequestSettings = formatNativeExportRequestSettings({ - videoCodec, - encoderPreference, - inputMode, - encodingMode: options.encodingMode, - width: options.width, - height: options.height, - frameRate: options.frameRate, - }); console.error( formatLogTs(), - `[native-export] Failed to start native video export session session=${sessionId || "unknown"} ${failedRequestSettings}:`, + `[native-export] Failed to start native video export session session=${sessionId || "unknown"} ${requestSettings}:`, error, ); return { diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs index b14081c3a..24299e2d5 100644 --- a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs @@ -9,6 +9,20 @@ // native --overlay descriptor must therefore use the physical frame count while // the returned metadata/summary keeps the logical count. Manifests without // effectiveFrameCount are fully dynamic layers and behave exactly as before. +// +// Two layer kinds are accepted: +// - "rgba" (default when `kind` is absent): a fixed-position raw RGBA overlay +// sidecar with a single `path`. Behavior is unchanged. Every layer carries +// its manifest `kind` and `order` (a safe-integer manifest order, otherwise +// a deterministic default), so classification and z-order survive JS-side +// filtering and the native descriptor keeps the renderer's global z-order. +// - "cursor-sprite": a tightly packed raw RGBA frame strip at `path` (one +// width*height*4 frame per output frame) whose per-frame top-left {x,y} +// position comes from a JSON `positionsPath` sidecar (exactly frameCount +// integer entries, top-down output pixels). Base x/y are 0 and ignored; +// positions are clamped to the output canvas and malformed/missing/truncated +// input fails closed so the cursor is never silently omitted. +// Unknown layer kinds are rejected rather than dropped. import { existsSync, readFileSync, statSync } from "node:fs"; import { resolve } from "node:path"; @@ -17,6 +31,64 @@ function fail(message) { throw new Error(message); } +// Clamps a top-left position so a partially off-canvas sprite keeps its visible +// part on screen. Sprite dimensions must already be validated against the +// canvas before clamping. +function clampPosition(position, spriteWidth, spriteHeight, outputWidth, outputHeight) { + const maxX = Math.max(0, outputWidth - spriteWidth); + const maxY = Math.max(0, outputHeight - spriteHeight); + return { + x: Math.max(0, Math.min(maxX, Number(position.x))), + y: Math.max(0, Math.min(maxY, Number(position.y))), + }; +} + +function readCursorSpritePositions( + positionsPath, + frameCount, + outputSize, + layerId, + spriteWidth, + spriteHeight, +) { + const resolvedPositionsPath = resolve(positionsPath); + if (!existsSync(resolvedPositionsPath)) { + fail(`Cursor-sprite layer ${layerId} positions do not exist: ${resolvedPositionsPath}`); + } + + let parsed; + try { + parsed = JSON.parse(readFileSync(resolvedPositionsPath, "utf8")); + } catch (error) { + fail(`Invalid cursor-sprite positions ${resolvedPositionsPath}: ${error.message}`); + } + const positions = Array.isArray(parsed) ? parsed : parsed?.positions; + if (!Array.isArray(positions)) { + fail( + `Cursor-sprite layer ${layerId} positions must be a JSON array of {x,y} objects: ` + + resolvedPositionsPath, + ); + } + if (positions.length !== frameCount) { + fail( + `Cursor-sprite layer ${layerId} positions must contain exactly one {x,y} per output ` + + `frame: expected ${frameCount}, received ${positions.length}`, + ); + } + + const { outputWidth, outputHeight } = outputSize; + return positions.map((position, index) => { + const x = Number(position?.x); + const y = Number(position?.y); + if (!Number.isSafeInteger(x) || !Number.isSafeInteger(y)) { + fail(`Cursor-sprite layer ${layerId} has a malformed position at frame ${index}`); + } + // Clamp the visible part of a partially off-canvas cursor rather than + // silently dropping it. + return clampPosition({ x, y }, spriteWidth, spriteHeight, outputWidth, outputHeight); + }); +} + export function readOverlayManifest(manifestPath, outputSize) { if (!manifestPath) { return []; @@ -40,6 +112,7 @@ export function readOverlayManifest(manifestPath, outputSize) { const layers = []; for (const layer of manifest.layers) { const id = typeof layer?.id === "string" ? layer.id : ""; + const kind = layer?.kind ?? "rgba"; const layerPath = typeof layer?.path === "string" ? layer.path : ""; const x = Number(layer?.x); const y = Number(layer?.y); @@ -63,6 +136,64 @@ export function readOverlayManifest(manifestPath, outputSize) { ) { fail(`Invalid overlay manifest layer ${id}: ${resolvedPath}`); } + if (kind === "cursor-sprite") { + if (width > outputWidth || height > outputHeight) { + fail(`Cursor-sprite layer ${id} exceeds the output canvas: ${resolvedPath}`); + } + if (effectiveFrameCount !== null) { + fail( + `Cursor-sprite layer ${id} does not support effectiveFrameCount: ${resolvedPath}`, + ); + } + // Base x/y are always 0 for a cursor-sprite; positions carry the + // per-frame top-left. + const positionsPath = + typeof layer?.positionsPath === "string" ? layer.positionsPath : ""; + if (!positionsPath) { + fail(`Cursor-sprite layer ${id} requires a positionsPath: ${resolvedPath}`); + } + const positions = readCursorSpritePositions( + positionsPath, + frameCount, + outputSize, + id, + width, + height, + ); + const resolvedLayerPath = resolve(layerPath); + if (!existsSync(resolvedLayerPath)) { + fail(`Cursor-sprite layer ${id} does not exist: ${resolvedLayerPath}`); + } + const expectedBytes = width * height * 4 * frameCount; + const stat = statSync(resolvedLayerPath); + if (stat.size < expectedBytes) { + fail( + `Cursor-sprite layer ${id} is truncated: expected at least ${expectedBytes} ` + + `bytes, received ${stat.size}`, + ); + } + // Cursor sprite layers blend above the fixed-position overlays; a + // manifest order takes precedence, otherwise default to a high value so + // the cursor stays sharp/topmost. + const order = Number.isSafeInteger(Number(layer?.order)) ? Number(layer.order) : 10000; + layers.push({ + id, + kind, + order, + path: resolvedLayerPath, + positionsPath: resolve(positionsPath), + x: 0, + y: 0, + width, + height, + frameCount, + positions, + }); + continue; + } + if (kind !== "rgba") { + fail(`Overlay manifest layer ${id} has an unexpected kind "${kind}": ${resolvedPath}`); + } if (effectiveFrameCount !== null) { // Mirror the renderer contract (validateNativeStaticLayoutOverlayLayer): // the physical sidecar count must be a positive integer no greater than @@ -91,8 +222,18 @@ export function readOverlayManifest(manifestPath, outputSize) { `Overlay layer ${id} is truncated: expected at least ${expectedBytes} bytes, received ${stat.size}`, ); } + // A manifest order takes precedence; otherwise default to the layer's + // position in the sorted manifest so relative z-order survives even when + // the producer omits the field (mirrors the native --overlay insertion + // default). Mixed rgba/cursor-sprite manifests keep ascending z-order and + // the cursor-sprite default (10000) stays above the fixed-position layers. + const order = Number.isSafeInteger(Number(layer?.order)) + ? Number(layer.order) + : layers.length; layers.push({ id, + kind, + order, path: resolvedLayerPath, x, y, diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs index df0b2db85..78547565a 100644 --- a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs @@ -86,6 +86,8 @@ describe("readOverlayManifest", () => { expect(layers).toEqual([ { id: "overlay-a", + kind: "rgba", + order: 0, path: sidecar, x: 0, y: 0, @@ -110,6 +112,8 @@ describe("readOverlayManifest", () => { expect(layers).toEqual([ { id: "overlay-a", + kind: "rgba", + order: 0, path: sidecar, x: 0, y: 0, @@ -292,3 +296,310 @@ describe("readOverlayManifest", () => { } }); }); + +// Cursor-sprite overlay kind. A cursor-sprite layer is a tightly packed raw +// RGBA frame strip (one width*height*4 frame per output frame) whose per-frame +// top-left {x,y} comes from a JSON positions sidecar (exactly frameCount +// entries, top-down output pixels). Base x/y are always 0; positions are +// clamped to keep the visible part of a partially off-canvas cursor on screen, +// and malformed/missing/truncated input fails closed (never silently omits). + +function cursorSpriteLayer(overrides = {}) { + return { + id: "cursor-sprite", + kind: "cursor-sprite", + path: "", + positionsPath: "", + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + ...overrides, + }; +} + +function writePositions(dir, positions, name = "cursor.positions.json") { + const positionsPath = join(dir, name); + writeFileSync(positionsPath, JSON.stringify(positions)); + return positionsPath; +} + +describe("cursor-sprite overlay layers", () => { + it("accepts a cursor-sprite layer with a per-frame positions sidecar", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, (_, index) => ({ x: index, y: 10 - index })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(1); + expect(layers[0]).toMatchObject({ + id: "cursor-sprite", + kind: "cursor-sprite", + path: sprite, + positionsPath, + x: 0, + y: 0, + width: 4, + height: 4, + frameCount: 10, + }); + expect(layers[0].positions).toEqual( + Array.from({ length: 10 }, (_, index) => ({ x: index, y: 10 - index })), + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("accepts a positions file wrapped in a { positions } object", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions(dir, { + positions: Array.from({ length: 10 }, (_, index) => ({ x: index, y: 0 })), + }); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].positions).toHaveLength(10); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("clamps a partially off-canvas cursor position instead of dropping it", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, (_, index) => ({ + x: index === 0 ? -5 : 1919, + y: index === 0 ? 5 : 1080, + })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers[0].positions[0]).toEqual({ x: 0, y: 5 }); + expect(layers[0].positions[9]).toEqual({ x: 1919 - 3, y: 1080 - 4 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a missing or malformed positions sidecar", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const missing = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath: join(dir, "nope.json") }), + ]); + expect(() => readOverlayManifest(missing, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite positions do not exist:", + ); + + const badJsonPath = join(dir, "bad.json"); + writeFileSync(badJsonPath, "{not json"); + const badJson = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath: badJsonPath }), + ]); + expect(() => readOverlayManifest(badJson, OUTPUT_SIZE)).toThrow( + "Invalid cursor-sprite positions", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a positions count that does not match the frame count", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 9 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite positions must contain exactly one {x,y} per output frame: expected 10, received 9", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a malformed (non-integer or negative) position", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positions = Array.from({ length: 10 }, () => ({ x: 0, y: 0 })); + positions[3] = { x: 0.5, y: 0 }; + const positionsPath = writePositions(dir, positions); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite has a malformed position at frame 3", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a truncated cursor-sprite frame strip", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 9, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + `Cursor-sprite layer cursor-sprite is truncated: expected at least ${ + FRAME_BYTES * 10 + } bytes, received ${FRAME_BYTES * 9}`, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects a cursor-sprite layer whose sprite exceeds the output canvas", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ path: sprite, positionsPath, width: 1921 }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite exceeds the output canvas:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects unexpected layer kinds rather than dropping them", () => { + const dir = makeTempDir(); + try { + const sidecar = writeSidecar(dir, FRAME_BYTES * 10); + const manifestPath = writeManifest(dir, [layer({ path: sidecar, kind: "unknown" })]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + 'Overlay manifest layer overlay-a has an unexpected kind "unknown":', + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("rejects effectiveFrameCount on a cursor-sprite layer", () => { + const dir = makeTempDir(); + try { + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 0, y: 0 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ + path: sprite, + positionsPath, + effectiveFrameCount: 3, + }), + ]); + expect(() => readOverlayManifest(manifestPath, OUTPUT_SIZE)).toThrow( + "Cursor-sprite layer cursor-sprite does not support effectiveFrameCount:", + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("mixes rgba and cursor-sprite layers in one manifest", () => { + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + layer({ id: "a", path: rgbaSidecar }), + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "a", kind: "rgba" }); + expect(layers[1]).toMatchObject({ id: "cursor", kind: "cursor-sprite" }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +// Layer classification and z-order regression: the reader must attach the +// manifest `kind`/`order` fields to every returned layer so the wrapper's +// kind filters never drop a layer and the native descriptor keeps the +// renderer-side global z-order (rgba layers included). + +describe("layer classification and z-order", () => { + it("attaches the manifest kind and order to every layer so classification and z-order survive", () => { + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + layer({ id: "bottom", path: rgbaSidecar, order: 5 }), + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath, order: 7 }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "bottom", kind: "rgba", order: 5 }); + expect(layers[1]).toMatchObject({ id: "cursor", kind: "cursor-sprite", order: 7 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("defaults rgba layers to a deterministic order when the manifest omits it", () => { + const dir = makeTempDir(); + try { + const firstSidecar = writeSidecar(dir, FRAME_BYTES * 10, "first.rgba"); + const secondSidecar = writeSidecar(dir, FRAME_BYTES * 10, "second.rgba"); + const manifestPath = writeManifest(dir, [ + layer({ id: "first", path: firstSidecar }), + layer({ id: "second", path: secondSidecar }), + ]); + const layers = readOverlayManifest(manifestPath, OUTPUT_SIZE); + expect(layers).toHaveLength(2); + expect(layers[0]).toMatchObject({ id: "first", kind: "rgba", order: 0 }); + expect(layers[1]).toMatchObject({ id: "second", kind: "rgba", order: 1 }); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs index 8cc9a14cf..33da26c9f 100644 --- a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs +++ b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs @@ -969,6 +969,17 @@ function resolveNativeProbePath() { } const nativeProbe = resolveNativeProbePath(); +// The --help capability probe output is stable for a given helper build; +// compute it once and reuse it for every feature check in this export instead +// of re-invoking the native binary once per overlay/temporal feature. +let nativeHelpCache = null; +function readNativeHelp() { + if (nativeHelpCache === null) { + nativeHelpCache = run(nativeProbe, ["--help"]).stdout; + } + return nativeHelpCache; +} + const baseName = basename(inputPath).replace(/\.[^.]+$/, ""); const webcamBaseName = webcamInput ? basename(webcamInput).replace(/\.[^.]+$/, "") @@ -1373,7 +1384,7 @@ if (temporalBlurSampleCount > 0) { // motion blur plan (mirror of the tiled-overlay probe below). Until then // temporal blur cannot be composited and the export fails fast instead of // silently dropping the effect. - const nativeHelp = run(nativeProbe, ["--help"]).stdout; + const nativeHelp = readNativeHelp(); if (!nativeHelp.includes("--temporal-blur-sample-count")) { fail( "unsupported-temporal-motion-blur: the native NVIDIA CUDA compositor does not support temporal zoom motion blur yet; " + @@ -1393,10 +1404,13 @@ if (temporalBlurSampleCount >= 3) { ); } else if (temporalBlurSampleCount > 0) { // The TS-side invariant rejects resolved plans below the minimum (3), but a - // direct wrapper invocation could still request 1-2 samples; never silently - // omit the effect without a diagnostic. - console.warn( - `[nvidia-cuda-export] Temporal zoom motion blur requested with ${temporalBlurSampleCount} sample(s), below the minimum of 3; omitting the effect (unsupported-temporal-motion-blur)`, + // direct wrapper invocation could still request 1-2 samples. The native + // compositor accepts 3..61 samples only, so fail fast with the established + // unsupported-result contract instead of a warning and a silently dropped + // effect (mirroring the unsupported-temporal-motion-blur fail above). + fail( + `unsupported-temporal-motion-blur: temporal zoom motion blur requested with ${temporalBlurSampleCount} sample(s), below the minimum of 3; ` + + "main.cu only consumes --temporal-blur-sample-count values in the supported 3..61 range.", ); } // The manifest may mix fixed-position rgba layers and cursor-sprite layers. @@ -1423,11 +1437,15 @@ if (rgbaOverlayLayers.length) { // carry the physical sidecar count (effectiveFrameCount when renderer // dedup truncated an identical suffix, otherwise the logical count). String(layer.effectiveFrameCount ?? layer.frameCount), + // Optional 7th argument is the renderer-side global z-order; the native + // compositor merges raw/tiled/cursor-sprite layers by this ascending + // value so a manifest order survives classification and filtering. + String(layer.order), ); } } if (cursorSpriteLayers.length) { - const nativeHelp = run(nativeProbe, ["--help"]).stdout; + const nativeHelp = readNativeHelp(); if (!nativeHelp.includes("--cursor-sprite")) { fail( "The native NVIDIA CUDA compositor does not support cursor-sprite overlays yet; " + @@ -1488,7 +1506,7 @@ const tiledOverlayMetrics = tiledOverlayLayers.map((layer) => { }; }); if (tiledOverlayLayers.length) { - const nativeHelp = run(nativeProbe, ["--help"]).stdout; + const nativeHelp = readNativeHelp(); if (!nativeHelp.includes("--tiled-overlay-manifest")) { fail( "The native NVIDIA CUDA compositor does not support tiled overlay manifests yet; " + diff --git a/src/components/video-editor/ExportSettingsMenu.tsx b/src/components/video-editor/ExportSettingsMenu.tsx index 402738b4c..94bd0eae8 100644 --- a/src/components/video-editor/ExportSettingsMenu.tsx +++ b/src/components/video-editor/ExportSettingsMenu.tsx @@ -1,6 +1,6 @@ import { DownloadSimple as Download, FilmSlate as Film, Image, Info } from "@phosphor-icons/react"; import { LayoutGroup, motion } from "motion/react"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Slider } from "@/components/ui/slider"; @@ -19,7 +19,10 @@ import type { GifSizePreset, } from "@/lib/exporter"; import { + clampCustomBitrateMbps, EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_H264_MAX_MBPS, + EXPORT_BITRATE_HEVC_MAX_MBPS, EXPORT_BITRATE_MIN_MBPS, GIF_FRAME_RATES, GIF_SIZE_PRESETS, @@ -109,22 +112,42 @@ export function ExportSettingsMenu({ }: ExportSettingsMenuProps) { const tSettings = useScopedT("settings"); const [bitrateDraft, setBitrateDraft] = useState(String(exportBitrateMbps)); + // Latest draft snapshot so the codec-switch clamp can re-clamp the draft + // without depending on (and re-running on) every keystroke. + const bitrateDraftRef = useRef(bitrateDraft); + const onExportBitrateMbpsChangeRef = useRef(onExportBitrateMbpsChange); + + useEffect(() => { + bitrateDraftRef.current = bitrateDraft; + }, [bitrateDraft]); + + useEffect(() => { + onExportBitrateMbpsChangeRef.current = onExportBitrateMbpsChange; + }, [onExportBitrateMbpsChange]); useEffect(() => { setBitrateDraft(String(exportBitrateMbps)); }, [exportBitrateMbps]); - const effectiveMaxMbps = exportVideoCodec === "hevc" ? 70 : 105; + const effectiveMaxMbps = + exportVideoCodec === "hevc" ? EXPORT_BITRATE_HEVC_MAX_MBPS : EXPORT_BITRATE_H264_MAX_MBPS; const commitBitrateDraft = () => { - const parsed = Number(bitrateDraft); - const clamped = Number.isFinite(parsed) - ? Math.min(effectiveMaxMbps, Math.max(EXPORT_BITRATE_MIN_MBPS, parsed)) - : Math.min(EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, effectiveMaxMbps); + const clamped = clampCustomBitrateMbps(Number(bitrateDraft), exportVideoCodec); setBitrateDraft(String(clamped)); onExportBitrateMbpsChange?.(clamped); }; + useEffect(() => { + // Re-clamp the draft and the emitted bitrate when the codec changes so an + // out-of-range draft (e.g. 105 Mbps H.264 value after switching to HEVC) + // does not survive the switch. Refs keep this effect keyed to codec + // switches only; it must not re-clamp the draft on every keystroke. + const clamped = clampCustomBitrateMbps(Number(bitrateDraftRef.current), exportVideoCodec); + setBitrateDraft(String(clamped)); + onExportBitrateMbpsChangeRef.current?.(clamped); + }, [exportVideoCodec]); + return (
{ ).toBe(getMp4ExportBitrate(second)); }); + it("caps auto-mode HEVC bitrates at the HEVC maximum", () => { + const autoOptions = { + width: 3840, + height: 2160, + frameRate: 30, + quality: "source" as const, + encodingMode: "quality" as const, + useModernNativeStaticLayout: true, + }; + // The uncapped auto heuristic exceeds 70 Mbps for 4K source exports. + expect(getMp4ExportBitrate(autoOptions)).toBe(72_000_000); + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + codec: "hevc", + ...autoOptions, + }), + ).toBe(70_000_000); + // h264 auto keeps the existing heuristic (no codec cap applied). + expect( + resolveExportBitrate({ + mode: "auto", + customMbps: 20, + codec: "h264", + ...autoOptions, + }), + ).toBe(72_000_000); + }); + it("custom mode converts Mbps to whole-number bps", () => { expect(customBitrateMbpsToBps(20)).toBe(20_000_000); expect(customBitrateMbpsToBps(12.5)).toBe(12_500_000); diff --git a/src/lib/exporter/exportBitrate.ts b/src/lib/exporter/exportBitrate.ts index 9718f6b2c..1d760b561 100644 --- a/src/lib/exporter/exportBitrate.ts +++ b/src/lib/exporter/exportBitrate.ts @@ -148,6 +148,14 @@ function getCodecCustomBitrateCapMbps(codec: ExportVideoCodec | undefined): numb } } +function getCodecAutoBitrateCapBps(codec: ExportVideoCodec | undefined): number { + if (codec === "hevc") { + return EXPORT_BITRATE_HEVC_MAX_MBPS * 1_000_000; + } + // h264 and unknown codecs keep the existing auto heuristic unchanged. + return Number.POSITIVE_INFINITY; +} + export function clampCustomBitrateMbps(mbps: number, codec?: ExportVideoCodec): number { if (!Number.isFinite(mbps) || Number.isNaN(mbps)) { return EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS; @@ -179,5 +187,5 @@ export function resolveExportBitrate(options: { if (options.mode === "custom") { return customBitrateMbpsToBps(options.customMbps, options.codec); } - return getMp4ExportBitrate(options); + return Math.min(getMp4ExportBitrate(options), getCodecAutoBitrateCapBps(options.codec)); } diff --git a/src/lib/exporter/index.ts b/src/lib/exporter/index.ts index 3838c19be..e004a596d 100644 --- a/src/lib/exporter/index.ts +++ b/src/lib/exporter/index.ts @@ -1,3 +1,21 @@ +export type { + CursorRect, + CursorSpriteCaptureResult, + CursorSpriteCapturerOptions, + CursorSpriteExpansion, + CursorSpriteRenderer, + CursorSpriteRoiResult, + CursorSpriteStripData, +} from "./cursorSpriteOverlay"; +export { + buildCursorSpriteRenderTransform, + CursorSpriteCapturer, + clampCursorRoiToCanvas, + DEFAULT_CURSOR_SPRITE_EXPANSION, + expandCursorBounds, + isValidCursorBounds, + resolveCursorRoi, +} from "./cursorSpriteOverlay"; export { clampCustomBitrateMbps, customBitrateMbpsToBps, @@ -17,10 +35,17 @@ export { resolveSupportedMp4EncoderPath, } from "./mp4Support"; export { VideoMuxer } from "./muxer"; -export type { NativeStaticLayoutOverlayLayer } from "./nativeStaticLayoutOverlays"; +export type { + NativeCursorSpriteOverlayLayer, + NativeCursorSpritePosition, + NativeStaticLayoutOverlayLayer, +} from "./nativeStaticLayoutOverlays"; export { + clampNativeCursorSpritePosition, getNativeStaticLayoutOverlayFrameByteSize, + isNativeCursorSpriteOverlayLayer, sortNativeStaticLayoutOverlayLayers, + validateNativeCursorSpriteOverlayLayer, validateNativeStaticLayoutOverlayLayer, } from "./nativeStaticLayoutOverlays"; export { StreamingVideoDecoder } from "./streamingDecoder"; @@ -48,6 +73,8 @@ export type { } from "./types"; export { EXPORT_BITRATE_DEFAULT_CUSTOM_MBPS, + EXPORT_BITRATE_H264_MAX_MBPS, + EXPORT_BITRATE_HEVC_MAX_MBPS, EXPORT_BITRATE_MAX_MBPS, EXPORT_BITRATE_MIN_MBPS, GIF_FRAME_RATES, diff --git a/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts index 5e07e7c65..4e8b1f3ab 100644 --- a/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts +++ b/src/lib/exporter/modernVideoExporter.overlayPreparation.test.ts @@ -29,6 +29,20 @@ const mocks = vi.hoisted(() => { height: 1080, }; const framesRendered: number[] = []; + // Cursor-sprite capture defaults to unavailable so the baked-cursor + // full-canvas sidecar fallback is the default behavior. Dedicated + // cursor-sprite tests override these to exercise the sprite success path. + // The defaults are re-applied in afterEach because mockReset() clears the + // implementations installed here by the hoisted factory. + const cursorSpriteDefaults = { + start: () => false, + capture: () => ({ + captured: false, + unavailableReason: "cursor sprite unavailable (mock)", + }), + finish: () => null, + cancel: () => {}, + }; return { framesRendered, @@ -39,16 +53,11 @@ const mocks = vi.hoisted(() => { frameRendererRenderOverlayFrame: vi.fn(async (timestampUs: number) => { framesRendered.push(timestampUs); }), - // Cursor-sprite capture defaults to unavailable so the baked-cursor - // full-canvas sidecar fallback is the default behavior. Dedicated - // cursor-sprite tests override these to exercise the sprite success path. - frameRendererStartCursorSpriteCapture: vi.fn(() => false), - frameRendererCaptureCursorSpriteFrame: vi.fn(() => ({ - captured: false, - unavailableReason: "cursor sprite unavailable (mock)", - })), - frameRendererFinishCursorSpriteCapture: vi.fn(() => null), - frameRendererCancelCursorSpriteCapture: vi.fn(() => {}), + frameRendererStartCursorSpriteCapture: vi.fn(cursorSpriteDefaults.start), + frameRendererCaptureCursorSpriteFrame: vi.fn(cursorSpriteDefaults.capture), + frameRendererFinishCursorSpriteCapture: vi.fn(cursorSpriteDefaults.finish), + frameRendererCancelCursorSpriteCapture: vi.fn(cursorSpriteDefaults.cancel), + cursorSpriteDefaults, }; }); @@ -241,10 +250,24 @@ describe("ModernVideoExporter native overlay preparation", () => { vi.clearAllMocks(); // Reset cursor-sprite mock implementations so the default (unavailable) // fallback applies unless a test explicitly opts into the sprite path. + // mockReset() clears the defaults installed by the hoisted factory, so + // re-apply them here for tests that rely on the unavailable fallback. mocks.frameRendererStartCursorSpriteCapture.mockReset(); mocks.frameRendererCaptureCursorSpriteFrame.mockReset(); mocks.frameRendererFinishCursorSpriteCapture.mockReset(); mocks.frameRendererCancelCursorSpriteCapture.mockReset(); + mocks.frameRendererStartCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.start, + ); + mocks.frameRendererCaptureCursorSpriteFrame.mockImplementation( + mocks.cursorSpriteDefaults.capture, + ); + mocks.frameRendererFinishCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.finish, + ); + mocks.frameRendererCancelCursorSpriteCapture.mockImplementation( + mocks.cursorSpriteDefaults.cancel, + ); vi.unstubAllGlobals(); }); diff --git a/src/lib/exporter/modernVideoExporter.progressDedup.test.ts b/src/lib/exporter/modernVideoExporter.progressDedup.test.ts new file mode 100644 index 000000000..504900b21 --- /dev/null +++ b/src/lib/exporter/modernVideoExporter.progressDedup.test.ts @@ -0,0 +1,150 @@ +import { beforeAll, describe, expect, it, vi } from "vitest"; +import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import type { ExportProgress } from "./types"; + +describe("ModernVideoExporter reportProgress preparing dedup", () => { + let ModernVideoExporter: typeof ModernVideoExporterClass; + + beforeAll(async () => { + ({ ModernVideoExporter } = await import("./modernVideoExporter")); + }, 30_000); + + it("delivers the first preparing signal and suppresses identical repeats for one export", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + + expect(emitted).toHaveLength(1); + expect(emitted[0]).toMatchObject({ + currentFrame: 0, + totalFrames: 100, + phase: "preparing", + percentage: 0, + }); + }); + + it("does not suppress progress that carries render or audio progress", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing", undefined, 0.5); + reporter(0, 100, "preparing", undefined, 0.6); + + // The first plain preparing is suppressed for the second, but the two + // audio-progress-bearing signals each still carry their distinct values. + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => p.audioProgress)).toEqual([undefined, 0.5, 0.6]); + }); + + it("emits again once the export moves on and a different total frame count starts", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(50, 100, "extracting"); + reporter(0, 120, "preparing"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.totalFrames, p.phase])).toEqual([ + [0, 100, "preparing"], + [50, 100, "extracting"], + [0, 120, "preparing"], + ]); + }); + + it("re-delivers the preparing signal after a non-preparing phase reuses the same total frame count", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + // A non-preparing event ends the preparing phase, so the watermark must not + // suppress a later preparing phase that reuses the same total frame count. + reporter(50, 100, "extracting"); + reporter(0, 100, "preparing"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.totalFrames, p.phase])).toEqual([ + [0, 100, "preparing"], + [50, 100, "extracting"], + [0, 100, "preparing"], + ]); + }); + + it("keeps distributing later phase progress after the preparing signal", async () => { + const emitted: ExportProgress[] = []; + const exporter = new ModernVideoExporter({ + onProgress: (progress) => emitted.push(progress), + } as never) as unknown as { + reportProgress: ( + currentFrame: number, + totalFrames: number, + phase: "preparing" | "extracting" | "finalizing" | "saving", + renderProgress?: number, + audioProgress?: number, + ) => void; + }; + + const reporter = exporter.reportProgress.bind(exporter); + reporter(0, 100, "preparing"); + reporter(0, 100, "preparing"); + reporter(10, 100, "extracting"); + reporter(20, 100, "extracting"); + + expect(emitted).toHaveLength(3); + expect(emitted.map((p) => [p.currentFrame, p.phase])).toEqual([ + [0, "preparing"], + [10, "extracting"], + [20, "extracting"], + ]); + }); +}); diff --git a/src/lib/exporter/modernVideoExporter.routeRejection.test.ts b/src/lib/exporter/modernVideoExporter.routeRejection.test.ts new file mode 100644 index 000000000..3ac22b8df --- /dev/null +++ b/src/lib/exporter/modernVideoExporter.routeRejection.test.ts @@ -0,0 +1,277 @@ +import { afterEach, beforeAll, describe, expect, it, vi } from "vitest"; +import type { ModernVideoExporter as ModernVideoExporterClass } from "./modernVideoExporter"; +import type { DecodedVideoInfo } from "./streamingDecoder"; + +const FRAME_BYTE_SIZE = 1920 * 1080 * 4; +const DEFAULT_FRAME_VALUE = 0xaa; + +const frameSource = vi.hoisted(() => { + return { + values: [] as number[], + videoFrameCall: 0, + readbackCall: 0, + fill: (frameIndex: number, buffer: Uint8Array | Uint8ClampedArray) => { + const value = frameSource.values[frameIndex] ?? DEFAULT_FRAME_VALUE; + buffer.fill(value); + }, + }; +}); + +const mocks = vi.hoisted(() => { + const rendererCanvas = { + width: 1920, + height: 1080, + }; + + return { + rendererCanvas, + frameRendererDestroy: vi.fn(), + frameRendererGetCanvas: vi.fn(() => rendererCanvas), + frameRendererInitialize: vi.fn(async () => {}), + frameRendererRenderOverlayFrame: vi.fn(async () => {}), + // Cursor-sprite capture defaults to unavailable so the baked-cursor + // full-canvas sidecar fallback is the default behavior. Dedicated + // cursor-sprite tests override these to exercise the sprite success path. + frameRendererStartCursorSpriteCapture: vi.fn(() => false), + frameRendererCaptureCursorSpriteFrame: vi.fn(() => ({ + captured: false, + unavailableReason: "cursor sprite unavailable (mock)", + })), + frameRendererFinishCursorSpriteCapture: vi.fn(() => null), + frameRendererCancelCursorSpriteCapture: vi.fn(() => {}), + }; +}); + +vi.mock("./modernFrameRenderer", () => ({ + FrameRenderer: vi.fn().mockImplementation(function () { + return { + destroy: mocks.frameRendererDestroy, + getCanvas: mocks.frameRendererGetCanvas, + initialize: mocks.frameRendererInitialize, + renderOverlayFrame: mocks.frameRendererRenderOverlayFrame, + startCursorSpriteCapture: mocks.frameRendererStartCursorSpriteCapture, + captureCursorSpriteFrame: mocks.frameRendererCaptureCursorSpriteFrame, + finishCursorSpriteCapture: mocks.frameRendererFinishCursorSpriteCapture, + cancelCursorSpriteCapture: mocks.frameRendererCancelCursorSpriteCapture, + }; + }), +})); + +class FakeVideoFrame { + constructor( + public readonly source: unknown, + public readonly init: { timestamp?: number } = {}, + ) {} + + async copyTo( + buffer: Uint8Array, + options: { format?: string; layout?: Array<{ offset: number; stride: number }> }, + ): Promise { + frameSource.fill(frameSource.videoFrameCall, buffer); + frameSource.videoFrameCall += 1; + void options; + } + + close(): void { + // no-op + } +} + +class FakeOffscreenCanvas { + width = 1920; + height = 1080; + + getContext(): { + clearRect: () => void; + drawImage: () => void; + getImageData: () => { data: Uint8ClampedArray }; + } { + return { + clearRect: () => undefined, + drawImage: () => undefined, + getImageData: () => { + const data = new Uint8ClampedArray(FRAME_BYTE_SIZE); + frameSource.fill(frameSource.readbackCall, data); + frameSource.readbackCall += 1; + return { data }; + }, + }; + } +} + +function createWindowStub() { + const streamBytes: Record = {}; + const electronAPI = { + openExportStream: vi.fn(async ({ extension }: { extension: string }) => { + const streamId = `overlay-${extension}`; + const tempPath = `C:/Temp/overlay.${extension}`; + streamBytes[streamId] = 0; + return { success: true, streamId, tempPath }; + }), + writeExportStreamChunk: vi.fn( + async (streamId: string, offset: number, chunk: Uint8Array) => { + streamBytes[streamId] = Math.max( + streamBytes[streamId] ?? 0, + offset + chunk.byteLength, + ); + return { success: true }; + }, + ), + closeExportStream: vi.fn(async (streamId: string) => { + const tempPath = `C:/Temp/overlay.${String(streamId).replace("overlay-", "")}`; + return { success: true, tempPath, bytesWritten: streamBytes[streamId] ?? 0 }; + }), + discardExportedTemp: vi.fn(async () => ({ success: true })), + nativeStaticLayoutExport: vi.fn(), + nativeStaticLayoutExportCancel: vi.fn(), + }; + vi.stubGlobal("window", { electronAPI }); + return electronAPI; +} + +function createExporter(overrides: Record = {}) { + return new ModernVideoExporter({ + videoUrl: "file:///recording.mp4", + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 8_000_000, + wallpaper: "#101010", + padding: 0, + borderRadius: 0, + backgroundBlur: 0, + shadowIntensity: 0, + showShadow: false, + cropRegion: { x: 0, y: 0, width: 1, height: 1 }, + experimentalNativeExport: true, + experimentalNvidiaCudaExport: true, + ...overrides, + } as never) as unknown as { + tryExportNativeStaticLayout: ( + videoInfo: DecodedVideoInfo, + audioPlan: unknown, + effectiveDurationSec: number, + totalFrames: number, + ) => Promise<{ success: boolean; tempFilePath?: string; error?: string } | null>; + nativeStaticLayoutSkipReason: string | null; + nativeStaticLayoutSkipReasons: string[]; + canUseNativeWebcamOwnership: () => boolean; + }; +} + +const videoInfo: DecodedVideoInfo = { + width: 1920, + height: 1080, + duration: 1, + streamDuration: 1, + frameRate: 30, + codec: "h264", + hasAudio: false, + audioCodec: null, + audioSampleRate: null, +}; + +let ModernVideoExporter: typeof ModernVideoExporterClass; + +describe("ModernVideoExporter native static-layout route rejection cleanup", () => { + beforeAll(async () => { + ({ ModernVideoExporter } = await import("./modernVideoExporter")); + }, 30_000); + + afterEach(() => { + frameSource.values = []; + frameSource.videoFrameCall = 0; + frameSource.readbackCall = 0; + vi.clearAllMocks(); + // Reset cursor-sprite mock implementations so the default (unavailable) + // fallback applies unless a test explicitly opts into the sprite path. + mocks.frameRendererStartCursorSpriteCapture.mockReset(); + mocks.frameRendererCaptureCursorSpriteFrame.mockReset(); + mocks.frameRendererFinishCursorSpriteCapture.mockReset(); + mocks.frameRendererCancelCursorSpriteCapture.mockReset(); + vi.unstubAllGlobals(); + }); + + it("discards the produced temp video when the native route cannot compose the cursor sprite", async () => { + vi.stubGlobal("VideoFrame", FakeVideoFrame); + vi.stubGlobal("OffscreenCanvas", FakeOffscreenCanvas); + const api = createWindowStub(); + mocks.frameRendererStartCursorSpriteCapture.mockReturnValue(true); + mocks.frameRendererCaptureCursorSpriteFrame.mockReturnValue({ + captured: true, + position: { x: 10, y: 20 }, + }); + mocks.frameRendererFinishCursorSpriteCapture.mockReturnValue({ + width: 32, + height: 32, + frameCount: 30, + frames: new Uint8Array(32 * 32 * 4 * 30), + positions: Array.from({ length: 30 }, (_, index) => ({ x: 10 + index, y: 20 })), + }); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/h264-cursor-sprite.mp4", + videoCodec: "h264", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + // Cursor motion blur disables native atlas ownership, so the cursor is + // prepared as a cursor-sprite ROI layer. A non-CUDA result route cannot + // compose that contract, so the successful native result is rejected; the + // produced temp video (potentially GBs) must not be left on disk for the + // session. + const exporter = createExporter({ + showCursor: true, + cursorMotionBlur: 1, + cursorTelemetry: [ + { timeMs: 0, cx: 0.25, cy: 0.35 }, + { timeMs: 500, cx: 0.4, cy: 0.45 }, + ], + }); + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-cursor-sprite-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-cursor-sprite-route"); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/h264-cursor-sprite.mp4"); + }); + + it("discards the produced temp video when the native route cannot draw the native-owned webcam", async () => { + const api = createWindowStub(); + api.nativeStaticLayoutExport.mockResolvedValue({ + success: true, + tempPath: "C:/Temp/h264-webcam-route.mp4", + videoCodec: "h264", + encoderPreference: "auto", + route: "cuda-overlay", + encoderName: "nvidia-cuda-compositor", + metrics: { chunkCount: 1, chunkDurationSec: 120, chunkExecMs: 0, chunks: [] }, + }); + + const exporter = createExporter(); + // The public strict-HEVC policy already refuses non-CUDA routes earlier, + // so force native webcam ownership here to exercise the explicit + // webcam-route invariant on a codec/preference combination that reaches it. + exporter.canUseNativeWebcamOwnership = () => true; + + const result = await exporter.tryExportNativeStaticLayout( + videoInfo, + { audioMode: "none" }, + 1, + 30, + ); + + expect(result).toBeNull(); + expect(exporter.nativeStaticLayoutSkipReason).toBe("unsupported-native-webcam-route"); + expect(exporter.nativeStaticLayoutSkipReasons).toContain("unsupported-native-webcam-route"); + expect(api.discardExportedTemp).toHaveBeenCalledWith("C:/Temp/h264-webcam-route.mp4"); + }); +}); diff --git a/src/lib/exporter/modernVideoExporter.ts b/src/lib/exporter/modernVideoExporter.ts index 9e3f04023..049061d62 100644 --- a/src/lib/exporter/modernVideoExporter.ts +++ b/src/lib/exporter/modernVideoExporter.ts @@ -4346,6 +4346,13 @@ export class ModernVideoExporter { ); this.nativeStaticLayoutSkipReason = routeSkipReason; this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs); + // discard it before falling back so it is not left on disk for the + // whole session. Best-effort: cleanup must never override the intended + // skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); restoreEncoderState(); return null; } @@ -4363,6 +4370,13 @@ export class ModernVideoExporter { ); this.nativeStaticLayoutSkipReason = routeSkipReason; this.nativeStaticLayoutSkipReasons = [routeSkipReason]; + // The native export already produced a temp video (potentially GBs); + // discard it before falling back so it is not left on disk for the + // whole session. Best-effort: cleanup must never override the intended + // skip reason or the null return. + await window.electronAPI + ?.discardExportedTemp?.(result.tempPath) + .catch(() => undefined); restoreEncoderState(); return null; } @@ -5288,6 +5302,13 @@ export class ModernVideoExporter { if (isIdenticalPreparingSignal) { this.lastPreparingTotalFrames = totalFrames; } + if (phase !== "preparing") { + // A non-preparing progress event ends the current preparing phase; reset + // the watermark so a later preparing phase that reuses the same total + // frame count still delivers its first signal instead of being suppressed + // against a stale total. + this.lastPreparingTotalFrames = null; + } const nowMs = this.getNowMs(); const elapsedSeconds = Math.max((nowMs - this.exportStartTimeMs) / 1000, 0.001); From 86cf212f00261265d0c7d9c1b329cda0714c5286 Mon Sep 17 00:00:00 2001 From: nmzpy <246990748+nmzpy@users.noreply.github.com> Date: Thu, 6 Aug 2026 19:43:52 +0300 Subject: [PATCH 11/11] fix(export): address CodeRabbit round-3 review - sort overlay layers by ascending order before building native wrapper args so cursor sprite stays above rgba layers - await prewarm teardown before native spawn; HEVC Hardware teardown failure hard-fails with noCpuFallback - mark native video export sessions finishing before write sequence and reject later frame writes --- electron/ipc/register/export.test.ts | 237 +++++++++++++++++- electron/ipc/register/export.ts | 66 ++++- .../overlayManifest.mjs | 13 + .../overlayManifest.test.mjs | 77 +++++- .../run-mp4-pipeline.mjs | 17 +- 5 files changed, 398 insertions(+), 12 deletions(-) diff --git a/electron/ipc/register/export.test.ts b/electron/ipc/register/export.test.ts index 54f0f091d..0a4b3802f 100644 --- a/electron/ipc/register/export.test.ts +++ b/electron/ipc/register/export.test.ts @@ -22,9 +22,15 @@ vi.mock("electron", () => ({ }, })); -vi.mock("../ffmpeg/binary", () => ({ getFfmpegBinaryPath: () => "ffmpeg" })); +vi.mock("../ffmpeg/binary", () => ({ + getFfmpegBinaryPath: () => path.join(process.cwd(), "recordly-missing-ffmpeg-binary"), +})); + +import { ipcMain } from "electron"; -import { moveExportedTempFile } from "./export"; +import * as nativeVideo from "../export/native-video"; +import { type NativeVideoExportSession, nativeVideoExportSessions } from "../export/native-video"; +import { moveExportedTempFile, registerExportHandlers } from "./export"; const tempDirs: string[] = []; @@ -79,3 +85,230 @@ describe("moveExportedTempFile", () => { await expect(fs.access(tempPath)).rejects.toThrow(); }); }); + +describe("registerExportHandlers native-video-export-start observability", () => { + function captureStartHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-start"); + expect(entry).toBeDefined(); + return entry?.[1] as (event: unknown, options: Record) => Promise; + } + + it("logs the incoming request settings before encoder resolution and on failure", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await handler( + { sender: {} }, + { + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 0, + encodingMode: "quality", + inputMode: "rawvideo", + videoCodec: "hevc", + encoderPreference: "hardware", + }, + ); + + // Missing ffmpeg binary forces the native encoder resolution to fail, so the + // handler still emits its pre-resolution request log before the failure log. + expect(result).toMatchObject({ success: false }); + + const logLines = logSpy.mock.calls.map((args) => String(args[1])); + const startRequest = logLines.find((line) => line.includes("Start request")); + expect(startRequest).toBeDefined(); + expect(startRequest).toMatch(/session=recordly-export-/); + expect(startRequest).toMatch(/codec=hevc/); + expect(startRequest).toMatch(/preference=hardware/); + expect(startRequest).toMatch(/input=rawvideo/); + expect(startRequest).toMatch(/mode=quality/); + expect(startRequest).toMatch(/1920x1080/); + expect(startRequest).toMatch(/fps=30/); + + const errorLines = errorSpy.mock.calls.map((args) => String(args[1])); + const failure = errorLines.find((line) => line.includes("Failed to start")); + expect(failure).toBeDefined(); + expect(failure).toMatch(/session=recordly-export-/); + expect(failure).toMatch(/codec=hevc/); + expect(failure).toMatch(/preference=hardware/); + expect(failure).toMatch(/1920x1080/); + }); + + it("logs effective defaults when codec/preference/input are omitted", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + const logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + + await handler( + { sender: {} }, + { + width: 640, + height: 360, + frameRate: 24, + bitrate: 0, + encodingMode: "fast", + }, + ); + + const logLines = logSpy.mock.calls.map((args) => String(args[1])); + const startRequest = logLines.find((line) => line.includes("Start request")); + expect(startRequest).toBeDefined(); + expect(startRequest).toMatch(/codec=h264/); + expect(startRequest).toMatch(/preference=auto/); + expect(startRequest).toMatch(/input=rawvideo/); + expect(startRequest).toMatch(/640x360/); + }); +}); + +describe("registerExportHandlers prewarm teardown hard-fail", () => { + function captureStartHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-start"); + expect(entry).toBeDefined(); + return entry?.[1] as (event: unknown, options: Record) => Promise; + } + + it("hard-fails HEVC Hardware exports with noCpuFallback when the prewarm teardown fails", async () => { + registerExportHandlers(); + const handler = captureStartHandler(); + vi.spyOn(console, "log").mockImplementation(() => {}); + vi.spyOn(console, "error").mockImplementation(() => {}); + vi.spyOn(console, "warn").mockImplementation(() => {}); + vi.spyOn(nativeVideo, "resolveNativeVideoEncoder").mockResolvedValue("hevc_nvenc"); + const cancelPrewarmSpy = vi + .spyOn(nativeVideo, "cancelInFlightCapabilityOnlyPrewarms") + .mockImplementation(() => { + throw new Error("capability-only prewarm child failed to terminate"); + }); + + const result = await handler( + { sender: {} }, + { + width: 1920, + height: 1080, + frameRate: 30, + bitrate: 0, + encodingMode: "quality", + inputMode: "rawvideo", + videoCodec: "hevc", + encoderPreference: "hardware", + }, + ); + + expect(result).toMatchObject({ success: false }); + const error = String((result as { error?: unknown }).error); + expect(error).toContain("noCpuFallback:true"); + expect(error).toContain("capability-only prewarm child failed to terminate"); + expect(cancelPrewarmSpy).toHaveBeenCalledTimes(1); + // The handler returns the strict hard-fail before the FFmpeg process spawn + // can ever run, so the export stops instead of continuing. + expect(error).toContain("before opening its encoder session"); + }); +}); + +describe("registerExportHandlers native-video-export-finish finalization state", () => { + function captureFinishHandler() { + const registrations = vi.mocked(ipcMain.handle).mock.calls; + const entry = registrations.find(([channel]) => channel === "native-video-export-finish"); + expect(entry).toBeDefined(); + return entry?.[1] as ( + event: unknown, + sessionId: string, + options?: unknown, + ) => Promise; + } + + function captureWriteFrameHandler() { + const registrations = vi.mocked(ipcMain.on).mock.calls; + const entry = registrations.find( + ([channel]) => channel === "native-video-export-write-frame-async", + ); + expect(entry).toBeDefined(); + return entry?.[1] as unknown as ( + event: { + sender: { send: (...args: unknown[]) => void; isDestroyed: () => boolean }; + }, + payload: { sessionId: string; requestId: number; frameData: Uint8Array }, + ) => void; + } + + it("rejects frame writes after finish has started", async () => { + registerExportHandlers(); + const finishHandler = captureFinishHandler(); + const writeFrameHandler = captureWriteFrameHandler(); + + let resolveWriteSequence!: () => void; + const writeSequence = new Promise((resolve) => { + resolveWriteSequence = resolve; + }); + + const sender = { send: vi.fn(), isDestroyed: () => false }; + const stdin = { + destroyed: false, + writableEnded: false, + writable: true, + writableLength: 0, + end: vi.fn(), + write: vi.fn(), + destroy: vi.fn(), + on: vi.fn(), + once: vi.fn(), + off: vi.fn(), + }; + const session = { + ffmpegProcess: { + stdin, + stderr: { on: vi.fn() }, + on: vi.fn(), + once: vi.fn(), + kill: vi.fn(), + }, + outputPath: path.join(os.tmpdir(), "recordly-finish-test.mp4"), + inputByteSize: 1920 * 1080 * 4, + inputMode: "rawvideo", + maxQueuedWriteBytes: 32 * 1024 * 1024, + stderrOutput: "", + encoderName: "hevc_nvenc", + processError: null, + stdinError: null, + terminating: false, + writeSequence, + completionPromise: Promise.resolve(), + sender: null, + pendingWriteRequestIds: new Set(), + framePort: null, + framePortReady: false, + nextFrameSequence: 0, + pendingFrameRequests: new Map(), + highestAcceptedFrameRequestId: -1, + } as unknown as NativeVideoExportSession; + + nativeVideoExportSessions.set("finish-test-session", session); + + const finishPromise = finishHandler(undefined, "finish-test-session"); + + writeFrameHandler( + { sender }, + { + sessionId: "finish-test-session", + requestId: 7, + frameData: new Uint8Array(1920 * 1080 * 4), + }, + ); + + expect(sender.send).toHaveBeenCalledWith("native-video-export-write-frame-result", { + sessionId: "finish-test-session", + requestId: 7, + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + + resolveWriteSequence(); + const result = await finishPromise; + expect(result).toMatchObject({ success: true }); + }); +}); diff --git a/electron/ipc/register/export.ts b/electron/ipc/register/export.ts index 4df40e7c2..57dc4558e 100644 --- a/electron/ipc/register/export.ts +++ b/electron/ipc/register/export.ts @@ -68,6 +68,16 @@ function getPartialExportDestinationPath(destinationPath: string) { const MAX_IN_MEMORY_EXPORT_BYTES = 0x7fffffff; +/** + * Native video export sessions that entered the finalization ("finish") phase. + * Marked before the finish handler awaits the write sequence so no further + * frame-write requests are accepted while already-accepted writes drain; the + * frame port is closed after the sequence settles. A WeakSet keeps the session + * contract in native-video.ts untouched and lets entries be collected once the + * session is removed from nativeVideoExportSessions. + */ +const finishingNativeVideoExportSessions = new WeakSet(); + /** * Structured, timestamped route/settings summary logged at the `native-video- * export-start` IPC boundary. Captures only high-level request settings (codec, @@ -389,9 +399,28 @@ export function registerExportHandlers() { // A capability-only prewarm child may still hold a brief NVENC probe // session; cancel it before this real export opens its real encoder - // session so they never contend for the GPU/hardware encoder. The - // prewarm is fire-and-forget (never awaited) so this never blocks IPC. - cancelInFlightCapabilityOnlyPrewarms(); + // session so they never contend for the GPU/hardware encoder. Await + // the teardown so no in-flight capability probe is still running when + // the FFmpeg process spawns below. + try { + await cancelInFlightCapabilityOnlyPrewarms(); + } catch (error) { + // Strict HEVC Hardware forbids every fallback: a prewarm that cannot + // be torn down must stop the export rather than let a stale NVENC + // probe contend with the real encoder session. + if (videoCodec === "hevc" && encoderPreference === "hardware") { + throw new Error( + `HEVC Hardware export requires tearing down the in-flight capability-only prewarm before opening its encoder session; prewarm teardown failed (noCpuFallback:true): ${error instanceof Error ? error.message : String(error)}`, + ); + } + // Non-strict routes keep the prewarm best-effort: a stale capability + // probe must never block a compatible export. + console.warn( + formatLogTs(), + `[native-export] Capability-only prewarm teardown failed session=${sessionId || "unknown"} ${requestSettings}:`, + error, + ); + } const ffmpegProcess = spawn(ffmpegPath, ffmpegArgs, { stdio: ["pipe", "ignore", "pipe"], @@ -637,6 +666,17 @@ export function registerExportHandlers() { return; } + if (finishingNativeVideoExportSessions.has(session)) { + sendNativeVideoExportFramePortError( + port, + sessionId, + "Native video export session is finishing; no more frames are accepted", + { fallbackAvailable: false }, + ); + port.close(); + return; + } + attachNativeVideoExportFramePort(sessionId, session, port, event.sender); }); @@ -683,6 +723,14 @@ export function registerExportHandlers() { return; } + if (finishingNativeVideoExportSessions.has(session)) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + return; + } + if ( session.inputMode !== "h264-stream" && frameDataList.some((frameData) => frameData.byteLength !== session.inputByteSize) @@ -751,6 +799,14 @@ export function registerExportHandlers() { return; } + if (finishingNativeVideoExportSessions.has(session)) { + settleNativeVideoExportWriteFrameRequest(sessionId, session, requestId, { + success: false, + error: "Native video export session is finishing; no more frames are accepted", + }); + return; + } + if ( session.inputMode !== "h264-stream" && frameData.byteLength !== session.inputByteSize @@ -789,6 +845,10 @@ export function registerExportHandlers() { return { success: false, error: "Invalid native export session" }; } + // Enter the finalization phase before settling the write sequence so + // subsequent frame-write requests are rejected while already-accepted + // writes drain; the frame port is closed after the sequence settles. + finishingNativeVideoExportSessions.add(session); try { await session.writeSequence; if ( diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs index 24299e2d5..fe6cd7f59 100644 --- a/electron/native/nvidia-cuda-compositor/overlayManifest.mjs +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.mjs @@ -245,3 +245,16 @@ export function readOverlayManifest(manifestPath, outputSize) { } return layers; } + +// Sorts overlay layers by ascending manifest z-order (order, then id) so the +// consumer's kind filters and the native descriptor always see the renderer's +// global z-order regardless of the manifest's physical order. Mirrors the sort +// used by the renderer-side native arg builders (order asc, then id +// localeCompare). Cursor-sprite layers keep their high default order (10000) +// when the producer omits the field, so they stay above fixed rgba layers even +// when the manifest lists them first. +export function sortOverlayLayersByOrder(layers) { + return [...layers].sort( + (left, right) => left.order - right.order || left.id.localeCompare(right.id), + ); +} diff --git a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs index 78547565a..7b36434ac 100644 --- a/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs +++ b/electron/native/nvidia-cuda-compositor/overlayManifest.test.mjs @@ -2,7 +2,7 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; -import { readOverlayManifest } from "./overlayManifest.mjs"; +import { readOverlayManifest, sortOverlayLayersByOrder } from "./overlayManifest.mjs"; // Contract for renderer-prepared transparent RGBA overlay sidecars: // - frameCount is the logical frame count (durationSec * frameRate). @@ -602,4 +602,79 @@ describe("layer classification and z-order", () => { rmSync(dir, { recursive: true, force: true }); } }); + + it("sorts a mixed manifest by (order, id) so the cursor-sprite stays above the rgba layers", () => { + // Regression for the CodeRabbit round-3 finding: readOverlayManifest + // preserves manifest order, so the consumer must sort by (order, id) + // before its kind filters. A deliberately non-sorted mixed manifest + // must still produce an ordering with the cursor-sprite layer above + // the lower-order rgba layer (and the id tie-break must be stable for + // equal orders). + const dir = makeTempDir(); + try { + const bottomSidecar = writeSidecar(dir, FRAME_BYTES * 10, "bottom.rgba"); + const topSidecar = writeSidecar(dir, FRAME_BYTES * 10, "top.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + // Non-sorted on purpose: cursor-sprite first, then the rgba + // layers in reverse order with equal orders to exercise the id + // tie-break. + cursorSpriteLayer({ id: "cursor-sprite", path: sprite, positionsPath, order: 3 }), + layer({ id: "z-rgba-top", path: topSidecar, order: 2 }), + layer({ id: "a-rgba-bottom", path: bottomSidecar, order: 2 }), + ]); + const layers = sortOverlayLayersByOrder(readOverlayManifest(manifestPath, OUTPUT_SIZE)); + expect(layers.map(({ id, kind, order }) => ({ id, kind, order }))).toEqual([ + { id: "a-rgba-bottom", kind: "rgba", order: 2 }, + { id: "z-rgba-top", kind: "rgba", order: 2 }, + { id: "cursor-sprite", kind: "cursor-sprite", order: 3 }, + ]); + // The consumer filters the sorted list into kind groups, so the + // cursor-sprite layer (order 3) ends up above every rgba layer + // (order 2) regardless of the manifest's physical order. + const rgba = layers.filter((layer) => layer.kind === "rgba"); + const cursor = layers.filter((layer) => layer.kind === "cursor-sprite"); + expect(rgba.map((layer) => layer.order)).toEqual([2, 2]); + expect(cursor.map((layer) => layer.order)).toEqual([3]); + expect(cursor[0].order).toBeGreaterThan(Math.max(...rgba.map((layer) => layer.order))); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + it("keeps a default-order cursor-sprite above rgba layers when the manifest lists the cursor first", () => { + // With omitted orders the reader defaults rgba layers to their manifest + // position (shifted by the cursor-sprite layer listed before them) and + // cursor-sprite layers to 10000; sorting must keep the cursor on top + // even though the manifest lists it before the rgba layers. + const dir = makeTempDir(); + try { + const rgbaSidecar = writeSidecar(dir, FRAME_BYTES * 10, "rgba.rgba"); + const sprite = writeSidecar(dir, FRAME_BYTES * 10, "cursor.rgba"); + const positionsPath = writePositions( + dir, + Array.from({ length: 10 }, () => ({ x: 1, y: 2 })), + ); + const manifestPath = writeManifest(dir, [ + cursorSpriteLayer({ id: "cursor", path: sprite, positionsPath }), + layer({ id: "first", path: rgbaSidecar }), + layer({ id: "second", path: rgbaSidecar }), + ]); + const layers = sortOverlayLayersByOrder(readOverlayManifest(manifestPath, OUTPUT_SIZE)); + expect(layers.map(({ id, kind, order }) => ({ id, kind, order }))).toEqual([ + { id: "first", kind: "rgba", order: 1 }, + { id: "second", kind: "rgba", order: 2 }, + { id: "cursor", kind: "cursor-sprite", order: 10000 }, + ]); + const rgba = layers.filter((layer) => layer.kind === "rgba"); + const cursor = layers.filter((layer) => layer.kind === "cursor-sprite"); + expect(cursor[0].order).toBeGreaterThan(Math.max(...rgba.map((layer) => layer.order))); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs index 33da26c9f..c103da29d 100644 --- a/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs +++ b/electron/native/nvidia-cuda-compositor/run-mp4-pipeline.mjs @@ -49,7 +49,7 @@ const ffmpegCommand = resolveToolCommand(["RECORDLY_FFMPEG_EXE"], "ffmpeg-static const ffprobeCommand = resolveToolCommand(["RECORDLY_FFPROBE_EXE"], "ffprobe-static", "ffprobe"); import { parseCursorTelemetrySamples, writeCursorSamplesFile } from "./cursorTelemetry.mjs"; -import { readOverlayManifest } from "./overlayManifest.mjs"; +import { readOverlayManifest, sortOverlayLayersByOrder } from "./overlayManifest.mjs"; import { shouldProbeSourcePts } from "./sourcePtsPlan.mjs"; import { readTiledOverlayManifest, @@ -1416,11 +1416,16 @@ if (temporalBlurSampleCount >= 3) { // The manifest may mix fixed-position rgba layers and cursor-sprite layers. // rgba layers keep the proven per-layer --overlay descriptor; cursor-sprite // layers are forwarded to the native cursor-sprite compositor route that owns -// the packed frame strip + per-frame positions validation. -const overlayLayers = readOverlayManifest(overlayManifest, { - outputWidth, - outputHeight, -}); +// the packed frame strip + per-frame positions validation. Layers are sorted +// by ascending (order, id) before the kind filters so mixed manifests keep the +// renderer's global z-order regardless of manifest order and cursor-sprite +// layers stay above the fixed rgba layers. +const overlayLayers = sortOverlayLayersByOrder( + readOverlayManifest(overlayManifest, { + outputWidth, + outputHeight, + }), +); const rgbaOverlayLayers = overlayLayers.filter((layer) => layer.kind === "rgba"); const cursorSpriteLayers = overlayLayers.filter((layer) => layer.kind === "cursor-sprite"); if (rgbaOverlayLayers.length) {