perf(recording): optimize the Windows D3D11 frame-conversion path

crates/enc-ffmpeg/src/video/h264.rs

@@ -1992,6 +1992,13 @@ fn get_bitrate(width: u32, height: u32, frame_rate: f32, bpp: f32) -> usize {
     ((pixels_per_second * bpp as f64) as usize).min(MAX_BITRATE)
 }
 
+/// Bitrate this module's encoders target for `bpp`, so a hardware encoder fed straight from the
+/// capture texture keeps the recording quality of the software path.
+pub fn bitrate_for_bpp(width: u32, height: u32, frame_rate: ffmpeg::Rational, bpp: f32) -> u32 {
+    let fps = frame_rate.0 as f32 / frame_rate.1.max(1) as f32;
+    get_bitrate(width, height, fps, bpp) as u32
+}
+
 const MAX_BITRATE: usize = 1_000_000_000;
 
 #[cfg(test)]

crates/enc-mediafoundation/src/d3d.rs

@@ -7,8 +7,8 @@ use windows::Win32::Graphics::{
         D3D_DRIVER_TYPE, D3D_DRIVER_TYPE_HARDWARE, D3D_DRIVER_TYPE_UNKNOWN, D3D_DRIVER_TYPE_WARP,
     },
     Direct3D11::{
-        D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG, D3D11_SDK_VERSION,
-        D3D11CreateDevice, ID3D11Device,
+        D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_CREATE_DEVICE_FLAG,
+        D3D11_CREATE_DEVICE_VIDEO_SUPPORT, D3D11_SDK_VERSION, D3D11CreateDevice, ID3D11Device,
     },
     Dxgi::{DXGI_ERROR_UNSUPPORTED, IDXGIDevice},
 };
@@ -70,7 +70,7 @@ fn create_d3d_device_on_pinned_adapter(
 pub fn create_d3d_device() -> Result<ID3D11Device> {
     let mut device = None;
     let flags = {
-        let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT;
+        let mut flags = D3D11_CREATE_DEVICE_BGRA_SUPPORT | D3D11_CREATE_DEVICE_VIDEO_SUPPORT;
         if cfg!(feature = "d3ddebug") {
             flags |= D3D11_CREATE_DEVICE_DEBUG;
         }
@@ -97,7 +97,12 @@ pub fn create_d3d_device() -> Result<ID3D11Device> {
     if let Err(error) = &result
         && error.code() == DXGI_ERROR_UNSUPPORTED
     {
-        result = create_d3d_device_with_type(D3D_DRIVER_TYPE_WARP, flags, &mut device);
+        // WARP rejects VIDEO_SUPPORT; a WARP device cannot hardware-encode anyway.
+        result = create_d3d_device_with_type(
+            D3D_DRIVER_TYPE_WARP,
+            D3D11_CREATE_DEVICE_BGRA_SUPPORT,
+            &mut device,
+        );
     }
     result?;
     device.ok_or_else(windows::core::Error::from_win32)

crates/enc-mediafoundation/src/video/video_processor.rs

@@ -18,7 +18,10 @@ use windows::{
                 ID3D11VideoContext, ID3D11VideoDevice, ID3D11VideoProcessor,
                 ID3D11VideoProcessorInputView, ID3D11VideoProcessorOutputView,
             },
-            Dxgi::Common::{DXGI_FORMAT, DXGI_RATIONAL, DXGI_SAMPLE_DESC},
+            Dxgi::Common::{
+                DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_R8G8B8A8_UNORM, DXGI_RATIONAL,
+                DXGI_SAMPLE_DESC,
+            },
         },
     },
     core::Interface,
@@ -218,6 +221,29 @@ impl VideoProcessor {
         &self.video_output_texture
     }
 
+    pub(super) fn set_bt709_color_space(&self, input_format: DXGI_FORMAT) {
+        let input_range = if matches!(
+            input_format,
+            DXGI_FORMAT_B8G8R8A8_UNORM | DXGI_FORMAT_R8G8B8A8_UNORM
+        ) {
+            D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_0_255
+        } else {
+            D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235
+        };
+        let output = D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
+            _bitfield: 1 | (1 << 2) | ((D3D11_VIDEO_PROCESSOR_NOMINAL_RANGE_16_235.0 as u32) << 4),
+        };
+        let input = D3D11_VIDEO_PROCESSOR_COLOR_SPACE {
+            _bitfield: 1 | (1 << 2) | ((input_range.0 as u32) << 4),
+        };
+        unsafe {
+            self.video_context
+                .VideoProcessorSetOutputColorSpace(&self.video_processor, &output);
+            self.video_context
+                .VideoProcessorSetStreamColorSpace(&self.video_processor, 0, &input);
+        }
+    }
+
     pub fn process_texture(
         &mut self,
         input_texture: &ID3D11Texture2D,

crates/frame-converter/examples/d3d11-benchmark.rs

@@ -0,0 +1,246 @@
+#[cfg(target_os = "windows")]
+mod benchmark {
+    use cap_frame_converter::{
+        AsyncConverterPool, ConversionConfig, ConverterPoolConfig, DropStrategy, FrameConverter,
+        SwscaleConverter,
+    };
+    use ffmpeg::{format::Pixel, frame};
+    use std::{
+        sync::Arc,
+        time::{Duration, Instant},
+    };
+
+    const OUTPUT_FORMAT: Pixel = Pixel::NV12;
+    const RESOLUTIONS: [(u32, u32); 2] = [(1280, 720), (1920, 1080)];
+    const WARMUP_FRAMES: usize = 10;
+    const MEASURED_FRAMES: usize = 120;
+    const POOL_MEASURED_FRAMES: usize = 60;
+    const MAX_IN_FLIGHT: usize = 8;
+
+    fn create_frame(format: Pixel, width: u32, height: u32) -> frame::Video {
+        let mut frame = frame::Video::new(format, width, height);
+        for plane in 0..frame.planes() {
+            for (index, byte) in frame.data_mut(plane).iter_mut().enumerate() {
+                *byte = ((index * 17 + plane * 31) % 256) as u8;
+            }
+        }
+        frame.set_pts(Some(0));
+        frame
+    }
+
+    fn input_format() -> Pixel {
+        match std::env::var("BENCH_INPUT").as_deref() {
+            Ok("nv12") => Pixel::NV12,
+            Ok("bgra") => Pixel::BGRA,
+            Ok("rgba") => Pixel::RGBA,
+            _ => Pixel::YUYV422,
+        }
+    }
+
+    fn conversion_config(width: u32, height: u32) -> ConversionConfig {
+        ConversionConfig::new(input_format(), width, height, OUTPUT_FORMAT, width, height)
+    }
+
+    fn benchmark_single_threaded(
+        converter: &dyn FrameConverter,
+        config: &ConversionConfig,
+    ) -> Duration {
+        for _ in 0..WARMUP_FRAMES {
+            let frame = create_frame(config.input_format, config.input_width, config.input_height);
+            let _ = converter.convert(frame).expect("warmup conversion failed");
+        }
+
+        let mut total = Duration::ZERO;
+        for _ in 0..MEASURED_FRAMES {
+            let frame = create_frame(config.input_format, config.input_width, config.input_height);
+            let start = Instant::now();
+            let _ = converter.convert(frame).expect("conversion failed");
+            total += start.elapsed();
+        }
+        total
+    }
+
+    fn pool_config(worker_count: usize) -> ConverterPoolConfig {
+        ConverterPoolConfig {
+            worker_count,
+            input_capacity: MAX_IN_FLIGHT * 2,
+            output_capacity: MAX_IN_FLIGHT * 2,
+            drop_strategy: DropStrategy::DropOldest,
+            frame_pool_capacity: MAX_IN_FLIGHT * 2,
+        }
+    }
+
+    fn measure_pool(pool: AsyncConverterPool, config: &ConversionConfig) -> (Duration, Duration) {
+        let mut frames = (0..WARMUP_FRAMES + POOL_MEASURED_FRAMES)
+            .map(|_| create_frame(config.input_format, config.input_width, config.input_height))
+            .collect::<Vec<_>>();
+
+        for sequence in 0..WARMUP_FRAMES {
+            let frame = frames.pop().expect("warmup frame");
+            pool.submit(frame, sequence as u64)
+                .expect("warmup submit failed");
+        }
+        for _ in 0..WARMUP_FRAMES {
+            pool.recv_timeout(Duration::from_secs(5))
+                .expect("warmup conversion timed out");
+        }
+
+        let start = Instant::now();
+        let mut next_sequence = 0usize;
+        let mut in_flight = 0usize;
+        let mut received = 0usize;
+        let mut conversion_time = Duration::ZERO;
+
+        while received < POOL_MEASURED_FRAMES {
+            while in_flight < MAX_IN_FLIGHT && next_sequence < POOL_MEASURED_FRAMES {
+                let frame = frames.pop().expect("measurement frame");
+                pool.submit(frame, next_sequence as u64)
+                    .expect("submit failed");
+                next_sequence += 1;
+                in_flight += 1;
+            }
+
+            let converted = pool
+                .recv_timeout(Duration::from_secs(5))
+                .expect("conversion timed out");
+            conversion_time += converted.conversion_duration;
+            in_flight -= 1;
+            received += 1;
+        }
+
+        (
+            start.elapsed(),
+            conversion_time / POOL_MEASURED_FRAMES as u32,
+        )
+    }
+
+    fn benchmark_pool(
+        converter: Arc<dyn FrameConverter>,
+        config: &ConversionConfig,
+        worker_count: usize,
+    ) -> (Duration, Duration) {
+        measure_pool(
+            AsyncConverterPool::new(converter, pool_config(worker_count)),
+            config,
+        )
+    }
+
+    fn benchmark_pool_from_config(
+        config: &ConversionConfig,
+        worker_count: usize,
+    ) -> (Duration, Duration) {
+        let pool = AsyncConverterPool::from_config(config.clone(), pool_config(worker_count))
+            .expect("converter pool from config");
+        measure_pool(pool, config)
+    }
+
+    fn print_pool_result(label: &str, (elapsed, conversion_time): (Duration, Duration)) {
+        let per_frame = elapsed / POOL_MEASURED_FRAMES as u32;
+        println!(
+            "{label}: wall {:?}/frame ({:.1} fps), convert {:?}",
+            per_frame,
+            1.0 / per_frame.as_secs_f64(),
+            conversion_time
+        );
+    }
+
+    fn worker_counts() -> Vec<usize> {
+        std::env::var("BENCH_WORKERS")
+            .ok()
+            .and_then(|raw| {
+                let counts = raw
+                    .split(',')
+                    .filter_map(|value| value.trim().parse().ok())
+                    .collect::<Vec<_>>();
+                (!counts.is_empty()).then_some(counts)
+            })
+            .unwrap_or_else(|| vec![1, 2, 4])
+    }
+
+    pub fn run() {
+        ffmpeg::init().expect("failed to initialize ffmpeg");
+        let single_threaded_only = std::env::var_os("BENCH_SINGLE_ONLY").is_some();
+        let pool_only = std::env::var_os("BENCH_POOL_ONLY").is_some();
+        let d3d11_only = std::env::var_os("BENCH_D3D11_ONLY").is_some();
+        let worker_counts = worker_counts();
+
+        println!("D3D11 frame conversion benchmark");
+        println!("Input: {:?}, output: {OUTPUT_FORMAT:?}\n", input_format());
+
+        for (width, height) in RESOLUTIONS {
+            let config = conversion_config(width, height);
+            println!("--- {width}x{height} ---");
+
+            let swscale = SwscaleConverter::new(config.clone()).expect("swscale converter");
+            let swscale_per_frame = if pool_only {
+                None
+            } else {
+                let swscale_time = benchmark_single_threaded(&swscale, &config);
+                let per_frame = swscale_time / MEASURED_FRAMES as u32;
+                println!(
+                    "swscale single-thread: {:>10?}/frame ({:>6.1} fps)",
+                    per_frame,
+                    1.0 / per_frame.as_secs_f64()
+                );
+                Some(per_frame)
+            };
+
+            match cap_frame_converter::D3D11Converter::new(config.clone()) {
+                Ok(d3d11) => {
+                    println!("d3d11 gpu: {}", d3d11.gpu_info().description);
+                    if let Some(swscale_per_frame) = swscale_per_frame {
+                        let d3d11_time = benchmark_single_threaded(&d3d11, &config);
+                        let d3d11_per_frame = d3d11_time / MEASURED_FRAMES as u32;
+                        println!(
+                            "d3d11 single-thread:  {:>10?}/frame ({:>6.1} fps, {:.1}x)",
+                            d3d11_per_frame,
+                            1.0 / d3d11_per_frame.as_secs_f64(),
+                            swscale_per_frame.as_secs_f64() / d3d11_per_frame.as_secs_f64()
+                        );
+                    }
+
+                    if !single_threaded_only {
+                        let d3d11: Arc<dyn FrameConverter> = Arc::new(d3d11);
+                        for &worker_count in &worker_counts {
+                            print_pool_result(
+                                &format!("d3d11 pool ({worker_count} workers, shared)"),
+                                benchmark_pool(Arc::clone(&d3d11), &config, worker_count),
+                            );
+                        }
+                        drop(d3d11);
+
+                        for &worker_count in &worker_counts {
+                            print_pool_result(
+                                &format!("d3d11 pool ({worker_count} workers, config)"),
+                                benchmark_pool_from_config(&config, worker_count),
+                            );
+                        }
+                    }
+                }
+                Err(error) => println!("d3d11 unavailable: {error}"),
+            }
+
+            if !single_threaded_only && !d3d11_only {
+                let swscale: Arc<dyn FrameConverter> = Arc::new(swscale);
+                for &worker_count in &worker_counts {
+                    print_pool_result(
+                        &format!("swscale pool ({worker_count} workers)"),
+                        benchmark_pool(Arc::clone(&swscale), &config, worker_count),
+                    );
+                }
+            }
+
+            println!();
+        }
+    }
+}
+
+#[cfg(target_os = "windows")]
+fn main() {
+    benchmark::run();
+}
+
+#[cfg(not(target_os = "windows"))]
+fn main() {
+    println!("D3D11 benchmark is only available on Windows");
+}