From a6aa0435c73a5d2b7d7557eb5b718299dff807a8 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Sun, 25 Oct 2026 20:32:52 +0100 Subject: [PATCH 1/7] perf(camera): remove per-frame copies from capture path Wrap Media Foundation buffers and CVPixelBuffer planes as refcounted AVFrames instead of allocating and copying each frame. Keep native Media Foundation buffers through the feed, run MF and DirectShow sample callbacks on bounded worker queues, and reuse an IOSurface-backed CVPixelBufferPool for VideoToolbox output. Add conversion counters and a synthetic 1080p frame-copy benchmark. --- Cargo.lock | 3 +- apps/desktop-gpui/Cargo.lock | 3 +- crates/camera-directshow/src/lib.rs | 110 ++++- crates/camera-ffmpeg/Cargo.toml | 6 +- .../examples/frame-copy-bench.rs | 111 ++++++ crates/camera-ffmpeg/src/lib.rs | 55 +++ crates/camera-ffmpeg/src/linux.rs | 8 +- crates/camera-ffmpeg/src/macos.rs | 211 +++++++++- crates/camera-ffmpeg/src/windows.rs | 375 +++++++++++++++++- crates/camera-mediafoundation/Cargo.toml | 1 - crates/camera-mediafoundation/src/lib.rs | 100 ++++- crates/camera-windows/src/lib.rs | 68 +++- crates/camera/src/lib.rs | 9 + crates/camera/src/windows.rs | 8 +- crates/frame-converter/Cargo.toml | 1 + crates/frame-converter/src/videotoolbox.rs | 320 ++++++--------- crates/recording/src/feeds/camera.rs | 66 +-- crates/recording/src/output_pipeline/win.rs | 60 ++- 18 files changed, 1241 insertions(+), 274 deletions(-) create mode 100644 crates/camera-ffmpeg/examples/frame-copy-bench.rs diff --git a/Cargo.lock b/Cargo.lock index f4294c43ea6..8208df72d5d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1225,6 +1225,7 @@ dependencies = [ "cap-camera", "cap-camera-avfoundation", "cap-camera-windows", + "cap-mediafoundation-utils", "cidre", "ffmpeg-next", "inquire", @@ -1241,7 +1242,6 @@ version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", "inquire", - "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", @@ -1472,6 +1472,7 @@ name = "cap-frame-converter" version = "0.1.0" dependencies = [ "cap-media-info", + "cidre", "crossbeam-channel", "ffmpeg-next", "flume", diff --git a/apps/desktop-gpui/Cargo.lock b/apps/desktop-gpui/Cargo.lock index d1fa933ee30..90b0ad76a07 100644 --- a/apps/desktop-gpui/Cargo.lock +++ b/apps/desktop-gpui/Cargo.lock @@ -1458,6 +1458,7 @@ dependencies = [ "cap-camera", "cap-camera-avfoundation", "cap-camera-windows", + "cap-mediafoundation-utils", "cidre", "ffmpeg-next", "thiserror 1.0.69", @@ -1472,7 +1473,6 @@ name = "cap-camera-mediafoundation" version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", - "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", @@ -1763,6 +1763,7 @@ name = "cap-frame-converter" version = "0.1.0" dependencies = [ "cap-media-info", + "cidre", "crossbeam-channel", "ffmpeg-next", "flume 0.11.1", diff --git a/crates/camera-directshow/src/lib.rs b/crates/camera-directshow/src/lib.rs index d2166053d68..3fb74fa08ad 100644 --- a/crates/camera-directshow/src/lib.rs +++ b/crates/camera-directshow/src/lib.rs @@ -8,7 +8,12 @@ use std::{ ops::Deref, os::windows::ffi::OsStringExt, ptr::{self, null, null_mut}, - sync::OnceLock, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel}, + }, + thread::{self, JoinHandle}, time::Duration, }; use tracing::*; @@ -460,7 +465,8 @@ impl VideoInputDevice { .SetFormat(&**format) .map_err(StartCapturingError::Other)?; - let sink_filter = SinkFilter::new(format.clone(), callback); + let (sink_filter, callback_worker) = SinkFilter::new(format.clone(), callback) + .map_err(StartCapturingError::CallbackWorker)?; let input_sink_pin = sink_filter .get_pin(0) @@ -515,6 +521,7 @@ impl VideoInputDevice { graph_builder, output_capture_pin: bound.output_pin, input_sink_pin, + callback_worker, }) } } @@ -525,6 +532,7 @@ pub struct CaptureHandle { graph_builder: IGraphBuilder, output_capture_pin: IPin, input_sink_pin: IPin, + callback_worker: CallbackWorker, } impl CaptureHandle { @@ -537,6 +545,8 @@ impl CaptureHandle { let _ = self.graph_builder.Disconnect(&self.input_sink_pin); } + drop(self.callback_worker); + Ok(()) } } @@ -553,6 +563,8 @@ pub enum StartCapturingError { ConfigureGraph(windows_core::Error), #[error("Run: {0}")] Run(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(std::io::Error), #[error("{0}")] Other(windows_core::Error), } @@ -628,13 +640,22 @@ pub struct SinkFilter { } impl SinkFilter { - pub fn new(desired_media_type: AMMediaType, callback: SinkCallback) -> ComObject { + pub(crate) fn new( + desired_media_type: AMMediaType, + callback: SinkCallback, + ) -> std::io::Result<(ComObject, CallbackWorker)> { + let (callback_tx, callback_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let dropped = Arc::new(AtomicU64::new(0)); + let callback_worker = + CallbackWorker::spawn(callback, callback_rx, Arc::new(AtomicBool::new(false)))?; + let this = SinkFilter { state: RefCell::new(State_Stopped), owning_graph: RefCell::new(None), input_pin: SinkInputPin { desired_media_type, - callback: RefCell::new(callback), + callback_tx, + dropped, current_media_type: Default::default(), connected_pin: Default::default(), owner: Default::default(), @@ -646,7 +667,7 @@ impl SinkFilter { // SAFETY: SinkFilter always implements IBaseFilter *this.input_pin.owner.borrow_mut() = Some(this.cast::().unwrap()); - this + Ok((this, callback_worker)) } fn no_of_pins(&self) -> u32 { @@ -814,7 +835,64 @@ pub struct CallbackData<'a> { pub perf_counter: i64, } -pub type SinkCallback = Box; +pub type SinkCallback = Box; + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct OwnedCallbackData { + sample: IMediaSample, + media_type: AMMediaType, + timestamp: Duration, + perf_counter: i64, +} + +unsafe impl Send for OwnedCallbackData {} + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: SinkCallback, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("dshow-camera-callback".to_string()) + .spawn(move || { + let _ = initialize_directshow(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(data) => callback(CallbackData { + sample: &data.sample, + media_type: &data.media_type, + timestamp: data.timestamp, + perf_counter: data.perf_counter, + }), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} #[implement(IPin, IMemInputPin)] struct SinkInputPin { @@ -823,7 +901,8 @@ struct SinkInputPin { current_media_type: RefCell, connected_pin: RefCell>, owner: RefCell>, - callback: RefCell, + callback_tx: SyncSender, + dropped: Arc, } // impl SinkInputPin { @@ -1029,7 +1108,7 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { } } - let media_type = self.current_media_type.borrow(); + let media_type = self.current_media_type.borrow().clone(); let length = unsafe { psample.GetActualDataLength() }; @@ -1046,12 +1125,19 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { unsafe { psample.GetTime(&mut start_time, &mut end_time) }?; - (self.callback.borrow_mut())(CallbackData { - sample: psample, - media_type: &media_type, + let data = OwnedCallbackData { + sample: psample.clone(), + media_type, timestamp: Duration::from_micros(start_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.callback_tx.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "DirectShow camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-ffmpeg/Cargo.toml b/crates/camera-ffmpeg/Cargo.toml index 6c98a31a488..6e7045baf25 100644 --- a/crates/camera-ffmpeg/Cargo.toml +++ b/crates/camera-ffmpeg/Cargo.toml @@ -16,9 +16,13 @@ cidre = { workspace = true } cap-camera-avfoundation = { path = "../camera-avfoundation" } [target.'cfg(windows)'.dependencies] -windows = { workspace = true } +windows = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Media_MediaFoundation", +] } windows-core = { workspace = true } cap-camera-windows = { path = "../camera-windows" } +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } [dev-dependencies] inquire = "0.7.5" diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs new file mode 100644 index 00000000000..9e9fe178b0d --- /dev/null +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -0,0 +1,111 @@ +#[cfg(windows)] +fn main() { + use cap_camera::CapturedFrame; + use cap_camera_ffmpeg::{CapturedFrameExt, conversion_stats}; + use cap_camera_windows::{Frame, PixelFormat}; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use ffmpeg::{format::Pixel, frame::Video}; + use std::time::{Duration, Instant}; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 1000; + + let len = WIDTH * HEIGHT * 3 / 2; + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + { + let mut lock = buffer.lock_for_write().expect("buffer lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("buffer length"); + + let frame = Frame::from_media_foundation( + buffer, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); + + for _ in 0..50 { + let frame = captured.as_ffmpeg().expect("warmup conversion"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = captured.as_ffmpeg().expect("conversion"); + std::hint::black_box(frame); + } + let zero_copy = start.elapsed(); + + let bytes = captured.native().bytes().expect("camera bytes"); + let start = Instant::now(); + for _ in 0..ITERATIONS { + let mut frame = Video::new(Pixel::NV12, WIDTH as u32, HEIGHT as u32); + let stride = frame.stride(0); + for row in 0..HEIGHT { + frame.data_mut(0)[row * stride..row * stride + WIDTH] + .copy_from_slice(&bytes[row * WIDTH..row * WIDTH + WIDTH]); + } + let uv_stride = frame.stride(1); + let uv_offset = WIDTH * HEIGHT; + for row in 0..HEIGHT / 2 { + frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH] + .copy_from_slice(&bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH]); + } + std::hint::black_box(frame); + } + let copy = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let buffer = captured.native().media_buffer().expect("native buffer"); + std::hint::black_box(buffer); + } + let buffer_clone = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let bytes = captured.native().bytes().expect("native bytes").to_vec(); + std::hint::black_box(bytes); + } + let bytes_copy = start.elapsed(); + + let stats = conversion_stats(); + println!( + "zero-copy: {:.3} ms/frame ({:.1} fps)", + zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / zero_copy.as_secs_f64() + ); + println!( + "copy: {:.3} ms/frame ({:.1} fps)", + copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / copy.as_secs_f64() + ); + println!( + "buffer clone: {:.3} ms/frame", + buffer_clone.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); + println!( + "bytes copy: {:.3} ms/frame", + bytes_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); + println!( + "stats: zero_copy={} copied={} avg_copy={:.3} us", + stats.zero_copy_frames, + stats.copied_frames, + stats.average_copy_micros() + ); +} + +#[cfg(not(windows))] +fn main() { + println!("frame-copy-bench requires Windows camera buffers"); +} diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index 2808e48b615..c8cbcbb1a6d 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -16,6 +16,61 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Debug, Default, Clone, Copy)] +pub struct ConversionStatsSnapshot { + pub zero_copy_frames: u64, + pub copied_frames: u64, + pub copy_nanos: u64, +} + +impl ConversionStatsSnapshot { + pub fn average_copy_micros(&self) -> f64 { + if self.copied_frames == 0 { + 0.0 + } else { + self.copy_nanos as f64 / self.copied_frames as f64 / 1_000.0 + } + } +} + +#[derive(Debug, Default)] +pub struct ConversionStats { + zero_copy_frames: AtomicU64, + copied_frames: AtomicU64, + copy_nanos: AtomicU64, +} + +impl ConversionStats { + pub fn record_zero_copy(&self) { + self.zero_copy_frames.fetch_add(1, Ordering::Relaxed); + } + + pub fn record_copy(&self, elapsed_nanos: u64) { + self.copied_frames.fetch_add(1, Ordering::Relaxed); + self.copy_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> ConversionStatsSnapshot { + ConversionStatsSnapshot { + zero_copy_frames: self.zero_copy_frames.load(Ordering::Relaxed), + copied_frames: self.copied_frames.load(Ordering::Relaxed), + copy_nanos: self.copy_nanos.load(Ordering::Relaxed), + } + } +} + +pub static CONVERSION_STATS: ConversionStats = ConversionStats { + zero_copy_frames: AtomicU64::new(0), + copied_frames: AtomicU64::new(0), + copy_nanos: AtomicU64::new(0), +}; + +pub fn conversion_stats() -> ConversionStatsSnapshot { + CONVERSION_STATS.snapshot() +} + pub trait CapturedFrameExt { /// Creates an ffmpeg video frame from the native frame. /// Only size, format, and data are set. diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index 03bbf2bdbeb..6738d6718e9 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -2,8 +2,8 @@ use cap_camera::{CapturedFrame, NativeFrameFormat}; use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; use std::cell::RefCell; -use crate::CapturedFrameExt; use crate::mjpeg_idct::idct_lowres; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -102,7 +102,11 @@ fn mjpeg_frame_covering( impl CapturedFrameExt for CapturedFrame<'_> { fn as_ffmpeg(&self) -> Result { let native = self.native(); - frame_from_bytes(native.bytes, native.format) + let started = std::time::Instant::now(); + let frame = frame_from_bytes(native.bytes, native.format)?; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } fn as_ffmpeg_covering( diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index 0857655cd66..b9892dbb74b 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -1,13 +1,28 @@ use cap_camera::CapturedFrame; use cap_camera_avfoundation::ImageBufExt; use cidre::*; -use ffmpeg::{format::Pixel, software::scaling}; +use ffmpeg::{ffi, format::Pixel, software::scaling}; use std::{ cell::RefCell, - sync::atomic::{AtomicBool, Ordering}, + ffi::c_void, + ptr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, }; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; + +type CVPixelBufferRef = *mut c_void; + +#[link(name = "CoreVideo", kind = "framework")] +unsafe extern "C" { + fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; + fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; + fn CVPixelBufferGetWidth(pixel_buffer: CVPixelBufferRef) -> usize; + fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; +} type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video); @@ -58,6 +73,181 @@ fn fourcc_to_pixel_format(fourcc: &str) -> Option { static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false); +#[link(name = "CoreVideo", kind = "framework")] +unsafe extern "C" { + fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; + fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; +} + +const CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 1; + +#[derive(Clone, Copy)] +struct SamplePlane { + ptr: *mut u8, + len: usize, + stride: usize, +} + +struct WrappedSampleBuf { + _image_buf: cidre::arc::R, + pixel_buffer: CVPixelBufferRef, + locked: bool, +} + +unsafe impl Send for WrappedSampleBuf {} +unsafe impl Sync for WrappedSampleBuf {} + +impl Drop for WrappedSampleBuf { + fn drop(&mut self) { + if self.locked { + unsafe { + CVPixelBufferUnlockBaseAddress(self.pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); + } + } + } +} + +unsafe extern "C" fn release_wrapped_sample_buf(opaque: *mut c_void, _data: *mut u8) { + if !opaque.is_null() { + drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + } +} + +fn sample_buf_planes( + image_buf: &cidre::cv::ImageBuf, + format: &str, +) -> Option<(Pixel, Vec, Option)> { + let (pixel, plane_count, range) = match format { + "2vuy" => (Pixel::UYVY422, 0, None), + "yuvs" => (Pixel::YUYV422, 0, None), + "BGRA" => (Pixel::BGRA, 0, None), + "ARGB" => (Pixel::ARGB, 0, None), + "24BG" => (Pixel::BGR24, 0, None), + "24RG" => (Pixel::RGB24, 0, None), + "L008" | "GRAY" => (Pixel::GRAY8, 0, None), + "L016" => (Pixel::GRAY16LE, 0, None), + "420v" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::MPEG)), + "420f" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::JPEG)), + "y420" => (Pixel::YUV420P, 3, None), + _ => return None, + }; + + if image_buf.plane_count() != plane_count { + return None; + } + + let mut planes = Vec::with_capacity(plane_count.max(1)); + if plane_count == 0 { + let pixel_buffer = image_buf as *const cidre::cv::ImageBuf as CVPixelBufferRef; + let ptr = unsafe { CVPixelBufferGetBaseAddress(pixel_buffer) }; + let stride = unsafe { CVPixelBufferGetBytesPerRow(pixel_buffer) }; + let height = unsafe { CVPixelBufferGetHeight(pixel_buffer) }; + if ptr.is_null() || stride == 0 || height == 0 { + return None; + } + planes.push(SamplePlane { + ptr, + len: stride.checked_mul(height)?, + stride, + }); + } else { + for plane in 0..plane_count { + let ptr = image_buf.plane_base_address(plane) as *mut u8; + let stride = image_buf.plane_bytes_per_row(plane); + let height = image_buf.plane_height(plane); + if ptr.is_null() || stride == 0 || height == 0 { + return None; + } + planes.push(SamplePlane { + ptr, + len: stride.checked_mul(height)?, + stride, + }); + } + } + + Some((pixel, planes, range)) +} + +fn wrap_sample_buf(sample_buf: &cidre::cm::SampleBuf) -> Option { + let image_buf = sample_buf.image_buf()?.retained(); + let format_desc = sample_buf.format_desc()?; + let format = cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()); + let pixel_buffer = (&*image_buf) as *const cidre::cv::ImageBuf as CVPixelBufferRef; + + if unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY) } != 0 { + return None; + } + + let Some((pixel, planes, range)) = sample_buf_planes(&image_buf, format) else { + unsafe { + CVPixelBufferUnlockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); + } + return None; + }; + + let owner = Arc::new(WrappedSampleBuf { + _image_buf: image_buf, + pixel_buffer, + locked: true, + }); + + let mut frame = ffmpeg::frame::Video::empty(); + let ptr = unsafe { frame.as_mut_ptr() }; + let format: ffi::AVPixelFormat = pixel.into(); + unsafe { + (*ptr).format = format as i32; + (*ptr).width = image_buf_width(&owner) as i32; + (*ptr).height = image_buf_height(&owner) as i32; + } + + let mut buffers = Vec::with_capacity(planes.len()); + for plane in &planes { + let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + plane.ptr, + plane.len, + Some(release_wrapped_sample_buf), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut Arc)); + for mut buffer in buffers { + ffi::av_buffer_unref(&mut buffer); + } + } + return None; + } + buffers.push(buffer); + } + + for (index, (buffer, plane)) in buffers.into_iter().zip(&planes).enumerate() { + unsafe { + (*ptr).buf[index] = buffer; + (*ptr).data[index] = plane.ptr; + (*ptr).linesize[index] = plane.stride as i32; + } + } + + if let Some(range) = range { + frame.set_color_range(range); + } + + Some(frame) +} + +fn image_buf_width(owner: &WrappedSampleBuf) -> usize { + unsafe { CVPixelBufferGetWidth(owner.pixel_buffer) } +} + +fn image_buf_height(owner: &WrappedSampleBuf) -> usize { + unsafe { CVPixelBufferGetHeight(owner.pixel_buffer) } +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) @@ -77,7 +267,13 @@ impl CapturedFrameExt for CapturedFrame { pub fn sample_buf_as_ffmpeg( sample_buf: &cm::SampleBuf, ) -> Result { - { + if let Some(frame) = wrap_sample_buf(sample_buf) { + CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + + let started = std::time::Instant::now(); + let frame = { let mut image_buf = sample_buf .image_buf() .map(|b| b.retained()) @@ -455,6 +651,9 @@ pub fn sample_buf_as_ffmpeg( } }; - Ok(res) - } + res + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 8f2a217a1e4..92a7326beb4 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -1,10 +1,16 @@ -use std::{cell::RefCell, collections::VecDeque}; +use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr, sync::Arc}; use cap_camera::CapturedFrame; use cap_camera_windows::PixelFormat; -use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; +use ffmpeg::{Packet, ffi, format::Pixel, frame::Video as FFVideo}; +use windows::{ + Win32::Media::MediaFoundation::{ + IMF2DBuffer, IMF2DBuffer2, IMFMediaBuffer, MF2DBuffer_LockFlags_Read, + }, + core::Interface, +}; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -229,6 +235,306 @@ fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize { if is_bottom_up { height - y - 1 } else { y } } +#[derive(Clone, Copy)] +struct MfPlane { + ptr: *mut u8, + len: usize, + stride: usize, +} + +enum MfLock { + TwoD2(IMF2DBuffer2), + TwoD(IMF2DBuffer), + Contiguous, +} + +struct WrappedMfBuffer { + _buffer: IMFMediaBuffer, + lock: MfLock, +} + +unsafe impl Send for WrappedMfBuffer {} +unsafe impl Sync for WrappedMfBuffer {} + +impl Drop for WrappedMfBuffer { + fn drop(&mut self) { + unsafe { + match &self.lock { + MfLock::TwoD2(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::TwoD(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::Contiguous => { + let _ = self._buffer.Unlock(); + } + } + } + } +} + +unsafe extern "C" fn release_wrapped_mf_buffer(opaque: *mut c_void, _data: *mut u8) { + if !opaque.is_null() { + drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + } +} + +fn mf_pixel_format(format: PixelFormat) -> Option { + Some(match format { + PixelFormat::NV12 => Pixel::NV12, + PixelFormat::NV21 => Pixel::NV21, + PixelFormat::YUYV422 => Pixel::YUYV422, + PixelFormat::UYVY422 => Pixel::UYVY422, + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => { + return None; + } + PixelFormat::ARGB | PixelFormat::RGB32 => Pixel::BGRA, + PixelFormat::RGB24 | PixelFormat::BGR24 => Pixel::BGR24, + PixelFormat::GRAY8 => Pixel::GRAY8, + PixelFormat::GRAY16 => Pixel::GRAY16LE, + PixelFormat::RGB565 => Pixel::RGB565LE, + PixelFormat::P010 => Pixel::P010LE, + }) +} + +fn mf_row_bytes(format: PixelFormat, width: usize) -> Option { + Some(match format { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::YUV420P | PixelFormat::GRAY8 => width, + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 + | PixelFormat::P010 => width.checked_mul(2)?, + PixelFormat::RGB24 | PixelFormat::BGR24 => width.checked_mul(3)?, + PixelFormat::ARGB | PixelFormat::RGB32 => width.checked_mul(4)?, + PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => return None, + }) +} + +fn mf_planes( + format: PixelFormat, + width: usize, + height: usize, + data: *mut u8, + len: usize, + stride: usize, +) -> Option> { + let row_bytes = mf_row_bytes(format, width)?; + if stride < row_bytes || height == 0 { + return None; + } + + let plane = |ptr: *mut u8, len: usize| MfPlane { ptr, len, stride }; + let y_len = stride.checked_mul(height)?; + + match format { + PixelFormat::NV12 | PixelFormat::NV21 => { + let uv_len = stride.checked_mul(height / 2)?; + if len < y_len.checked_add(uv_len)? { + return None; + } + Some(vec![ + plane(data, y_len), + plane(unsafe { data.add(y_len) }, uv_len), + ]) + } + PixelFormat::P010 => { + let uv_len = stride.checked_mul(height / 2)?; + if len < y_len.checked_add(uv_len)? { + return None; + } + Some(vec![ + plane(data, y_len), + plane(unsafe { data.add(y_len) }, uv_len), + ]) + } + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::ARGB + | PixelFormat::RGB32 + | PixelFormat::RGB24 + | PixelFormat::BGR24 + | PixelFormat::GRAY8 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 => { + if len < y_len { + return None; + } + Some(vec![plane(data, y_len)]) + } + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => None, + } +} + +fn lock_mf_buffer( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option<(WrappedMfBuffer, Vec, Pixel)> { + let pixel = mf_pixel_format(format)?; + + unsafe { + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + let mut start = ptr::null_mut(); + let mut len = 0u32; + if buffer_2d + .Lock2DSize( + MF2DBuffer_LockFlags_Read, + &mut scanline, + &mut pitch, + &mut start, + &mut len, + ) + .is_ok() + { + if !start.is_null() + && pitch > 0 + && len > 0 + && let Some(planes) = + mf_planes(format, width, height, start, len as usize, pitch as usize) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD2(buffer_2d), + }, + planes, + pixel, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + if buffer_2d.Lock2D(&mut scanline, &mut pitch).is_ok() { + if !scanline.is_null() + && pitch > 0 + && let Ok(len) = buffer_2d.GetContiguousLength() + && let Some(planes) = mf_planes( + format, + width, + height, + scanline, + len as usize, + pitch as usize, + ) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD(buffer_2d), + }, + planes, + pixel, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + let mut data = ptr::null_mut(); + let mut len = 0u32; + if buffer.Lock(&mut data, None, Some(&mut len)).is_ok() { + if !data.is_null() + && len > 0 + && let Some(row_bytes) = mf_row_bytes(format, width) + && let Some(planes) = + mf_planes(format, width, height, data, len as usize, row_bytes) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::Contiguous, + }, + planes, + pixel, + )); + } + + let _ = buffer.Unlock(); + } + } + + None +} + +pub fn media_buffer_as_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, +) -> Option { + if is_bottom_up || width == 0 || height == 0 { + return None; + } + + let (owner, planes, pixel) = lock_mf_buffer(buffer, format, width, height)?; + wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) +} + +fn wrap_mf_frame( + planes: &[MfPlane], + pixel: Pixel, + width: u32, + height: u32, + owner: Arc, +) -> Option { + let mut frame = FFVideo::empty(); + let ptr = unsafe { frame.as_mut_ptr() }; + let format: ffi::AVPixelFormat = pixel.into(); + unsafe { + (*ptr).format = format as i32; + (*ptr).width = width as i32; + (*ptr).height = height as i32; + } + + let mut buffers = Vec::with_capacity(planes.len()); + for plane in planes { + let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + plane.ptr, + plane.len, + Some(release_wrapped_mf_buffer), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut Arc)); + for mut buffer in buffers { + ffi::av_buffer_unref(&mut buffer); + } + } + return None; + } + + buffers.push(buffer); + } + + for (index, (buffer, plane)) in buffers.into_iter().zip(planes).enumerate() { + unsafe { + (*ptr).buf[index] = buffer; + (*ptr).data[index] = plane.ptr; + (*ptr).linesize[index] = plane.stride as i32; + } + } + + Some(frame) +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg_covering( &self, @@ -248,13 +554,22 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } + if let Some(buffer) = native.media_buffer() + && let Some(frame) = + media_buffer_as_ffmpeg(&buffer, native.pixel_format, width, height, is_bottom_up) + { + CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + let bytes = native.bytes().map_err(AsFFmpegError::FailedToGetBytes)?; if bytes.is_empty() { return Err(AsFFmpegError::Empty); } - Ok(match native.pixel_format { + let started = std::time::Instant::now(); + let frame = match native.pixel_format { PixelFormat::YUV420P => { let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32); @@ -510,6 +825,56 @@ impl CapturedFrameExt for CapturedFrame { ff_frame } PixelFormat::H264 => decode_h264(&bytes)?, - }) + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) + } +} + +#[cfg(test)] +mod zero_copy_tests { + use super::*; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + #[test] + fn nv12_media_buffer_is_wrapped_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for row in 0..height { + for column in 0..width { + lock[row * stride + column] = (row + column) as u8; + } + } + let uv = stride * height; + for row in 0..height / 2 { + for column in 0..width { + lock[uv + row * stride + column] = (row * 3 + column) as u8; + } + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = media_buffer_as_ffmpeg(&buffer, PixelFormat::NV12, width, height, false) + .expect("zero-copy frame"); + + assert_eq!(frame.format(), Pixel::NV12); + assert_eq!(frame.width(), width as u32); + assert_eq!(frame.height(), height as u32); + assert_eq!(frame.stride(0), stride); + assert_eq!(frame.stride(1), stride); + assert_eq!(frame.data(0)[0], 0); + assert_eq!(frame.data(0)[stride + 1], 2); + assert_eq!(frame.data(1)[0], 0); + assert_eq!(frame.data(1)[stride + 2], 5); } } diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 36ee6fdd9d0..0bee6fe27df 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -7,7 +7,6 @@ license = "MIT" [dependencies] tracing.workspace = true thiserror.workspace = true -parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index 246b1c311e9..e09fa426e21 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -2,7 +2,6 @@ #![allow(non_snake_case)] use cap_mediafoundation_utils::*; -use parking_lot::Mutex; use std::{ ffi::OsString, fmt::Display, @@ -11,9 +10,13 @@ use std::{ os::windows::ffi::OsStringExt, slice::from_raw_parts, sync::{ - OnceLock, - mpsc::{Receiver, Sender, channel}, + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{ + Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError, channel, sync_channel, + }, }, + thread::{self, JoinHandle}, time::Duration, }; use windows::Win32::{ @@ -144,6 +147,8 @@ pub enum StartCapturingError { ConfigureSink(windows_core::Error), #[error("StartPreview: {0}")] StartPreview(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(#[from] std::io::Error), } impl Device { @@ -233,7 +238,7 @@ impl Device { pub fn start_capturing( &self, requested_format: &IMFMediaType, - callback: Box, + callback: Box, ) -> Result { let media_source = self .media_source() @@ -252,9 +257,14 @@ impl Device { .map_err(StartCapturingError::CreateEngine)?; let (event_tx, event_rx) = channel(); + let (sample_tx, sample_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let dropped = Arc::new(AtomicU64::new(0)); + let callback_worker = + CallbackWorker::spawn(callback, sample_rx, Arc::new(AtomicBool::new(false)))?; let video_callback = VideoCallback { event_tx, - sample_callback: Mutex::new(callback), + samples: sample_tx, + dropped, } .into_object(); @@ -375,7 +385,11 @@ impl Device { wait_for_event(&event_rx, CaptureEngineEventVariant::PreviewStarted) .map_err(|v| StartCapturingError::StartPreview(v.into()))?; - Ok(CaptureHandle { engine, event_rx }) + Ok(CaptureHandle { + engine: Some(engine), + event_rx, + callback_worker, + }) } } } @@ -408,8 +422,9 @@ fn retry_on_invalid_request( } pub struct CaptureHandle { + engine: Option, event_rx: Receiver, - engine: IMFCaptureEngine, + callback_worker: CallbackWorker, } impl CaptureHandle { @@ -417,8 +432,14 @@ impl CaptureHandle { &self.event_rx } - pub fn stop_capturing(self) -> windows_core::Result<()> { - unsafe { self.engine.StopPreview() } + pub fn stop_capturing(mut self) -> windows_core::Result<()> { + let result = if let Some(engine) = self.engine.take() { + unsafe { engine.StopPreview() } + } else { + Ok(()) + }; + drop(self.callback_worker); + result } } @@ -570,10 +591,56 @@ pub struct CallbackData { pub perf_counter: i64, } +unsafe impl Send for CallbackData {} + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: Box, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("mf-camera-callback".to_string()) + .spawn(move || { + cap_mediafoundation_utils::thread_init(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(sample) => callback(sample), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + #[implement(IMFCaptureEngineOnSampleCallback, IMFCaptureEngineOnEventCallback)] struct VideoCallback { event_tx: Sender, - sample_callback: Mutex>, + samples: SyncSender, + dropped: Arc, } impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { @@ -585,15 +652,20 @@ impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { return Ok(()); }; - let mut callback = self.sample_callback.lock(); - let sample_time = unsafe { sample.GetSampleTime() }?; - (callback)(CallbackData { + let data = CallbackData { sample: sample.clone(), timestamp: Duration::from_micros(sample_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.samples.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "Media Foundation camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index b1004ae2d6c..526a279ffd6 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -236,8 +236,10 @@ impl VideoDeviceInfo { .max_by_key(|(w, h)| (*w as u64) * (*h as u64)) } - pub fn find_best_format(&self, preference: &FormatPreference) -> Option { - let formats = self.formats(); + fn best_format_in( + formats: &[VideoFormat], + preference: &FormatPreference, + ) -> Option { if formats.is_empty() { return None; } @@ -267,11 +269,19 @@ impl VideoDeviceInfo { format_priority + resolution_score + fps_score }; - formats.into_iter().max_by_key(score_format) + formats + .iter() + .max_by_key(|format| score_format(format)) + .cloned() + } + + pub fn find_best_format(&self, preference: &FormatPreference) -> Option { + Self::best_format_in(&self.formats(), preference) } pub fn find_format_with_fallback(&self, preference: &FormatPreference) -> Option { - if let Some(format) = self.find_best_format(preference) { + let formats = self.formats(); + if let Some(format) = Self::best_format_in(&formats, preference) { return Some(format); } @@ -284,7 +294,6 @@ impl VideoDeviceInfo { PixelFormat::YUV420P, ]; - let formats = self.formats(); for fallback_pixel_format in fallback_formats { if let Some(format) = formats .iter() @@ -319,7 +328,7 @@ impl VideoDeviceInfo { pub fn start_capturing( self, format: &VideoFormatInner, - mut callback: impl FnMut(Frame) + 'static, + mut callback: impl FnMut(Frame) + Send + 'static, ) -> Result { let res = match (self.inner, &format) { ( @@ -327,6 +336,10 @@ impl VideoDeviceInfo { VideoFormatInner::MediaFoundation(mf_format), ) => { let format = VideoFormat::new_mf(mf_format.clone())?; + let width = format.width() as usize; + let height = format.height() as usize; + let is_bottom_up = format.is_bottom_up; + let pixel_format = format.pixel_format; let handle = device.start_capturing( mf_format, @@ -340,10 +353,10 @@ impl VideoDeviceInfo { callback(Frame { inner: FrameInner::MediaFoundation(buffer), - width: format.width() as usize, - height: format.height() as usize, - is_bottom_up: format.is_bottom_up, - pixel_format: format.pixel_format, + width, + height, + is_bottom_up, + pixel_format, timestamp: data.timestamp, perf_counter: data.perf_counter, }) @@ -476,6 +489,41 @@ impl Frame { FrameInner::MediaFoundation(buffer) => Ok(FrameBytes::MediaFoundation(buffer.lock()?)), } } + + pub fn media_buffer(&self) -> Option { + match &self.inner { + FrameInner::MediaFoundation(buffer) => Some(buffer.clone()), + FrameInner::DirectShow(_) => None, + } + } + + pub fn directshow_sample(&self) -> Option { + match &self.inner { + FrameInner::DirectShow(sample) => Some(sample.clone()), + FrameInner::MediaFoundation(_) => None, + } + } + + #[doc(hidden)] + pub fn from_media_foundation( + buffer: IMFMediaBuffer, + pixel_format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, + timestamp: Duration, + perf_counter: i64, + ) -> Self { + Self { + pixel_format, + width, + height, + is_bottom_up, + timestamp, + perf_counter, + inner: FrameInner::MediaFoundation(buffer), + } + } } pub enum FrameBytes<'a> { diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index dd160277749..74f64a642eb 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -241,6 +241,15 @@ impl CapturedFrame { pub fn native(&self) -> &NativeCapturedFrame { &self.native } + + #[cfg(windows)] + #[doc(hidden)] + pub fn from_windows_frame(frame: cap_camera_windows::Frame, timestamp: Duration) -> Self { + Self { + native: NativeCapturedFrame::from_frame(frame), + timestamp, + } + } } #[cfg(target_os = "linux")] diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index a99d3d7b81f..07d72fa184d 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -64,12 +64,18 @@ impl ModelID { #[derive(Debug)] pub struct NativeCapturedFrame(cap_camera_windows::Frame); +impl NativeCapturedFrame { + pub(crate) fn from_frame(frame: cap_camera_windows::Frame) -> Self { + Self(frame) + } +} + pub type NativeCaptureHandle = WindowsCaptureHandle; pub(super) fn start_capturing_impl( camera: &CameraInfo, format: Format, - mut callback: impl FnMut(CapturedFrame) + 'static, + mut callback: impl FnMut(CapturedFrame) + Send + 'static, ) -> Result { let device = find_device(camera)?.ok_or(StartCapturingError::DeviceNotFound)?; diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index 3d363ef7098..be104b4593f 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -19,6 +19,7 @@ parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "macos")'.dependencies] +cidre = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows = { workspace = true, features = [ diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index 8d4d8e28d87..e0a104cd95e 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -1,4 +1,5 @@ use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; +use cidre::{arc, cf, cv, os, vt}; use ffmpeg::{format::Pixel, frame}; use parking_lot::Mutex; use std::{ @@ -7,100 +8,43 @@ use std::{ sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; -type CFAllocatorRef = *const c_void; -type CFDictionaryRef = *const c_void; type CVPixelBufferRef = *mut c_void; -type VTPixelTransferSessionRef = *mut c_void; -type OSStatus = i32; - -const K_CV_RETURN_SUCCESS: i32 = 0; - -const K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8: u32 = 0x79757679; -const K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE: u32 = 0x34323076; -const K_CV_PIXEL_FORMAT_TYPE_2VUY: u32 = 0x32767579; -const K_CV_PIXEL_FORMAT_TYPE_32_BGRA: u32 = 0x42475241; -const K_CV_PIXEL_FORMAT_TYPE_32_ARGB: u32 = 0x00000020; - -#[link(name = "CoreFoundation", kind = "framework")] -unsafe extern "C" { - fn CFRelease(cf: *const c_void); -} #[link(name = "CoreVideo", kind = "framework")] unsafe extern "C" { - fn CVPixelBufferCreate( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferCreateWithBytes( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - base_address: *mut c_void, - bytes_per_row: usize, - release_callback: *const c_void, - release_ref_con: *const c_void, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferRelease(pixel_buffer: CVPixelBufferRef); - - fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - - fn CVPixelBufferGetBaseAddressOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> *mut u8; - fn CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetHeightOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetPlaneCount(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; - fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; -} - -#[link(name = "VideoToolbox", kind = "framework")] -unsafe extern "C" { - fn VTPixelTransferSessionCreate( - allocator: CFAllocatorRef, - pixel_transfer_session_out: *mut VTPixelTransferSessionRef, - ) -> OSStatus; - - fn VTPixelTransferSessionInvalidate(session: VTPixelTransferSessionRef); - - fn VTPixelTransferSessionTransferImage( - session: VTPixelTransferSessionRef, - source_buffer: CVPixelBufferRef, - destination_buffer: CVPixelBufferRef, - ) -> OSStatus; + fn CVPixelBufferGetBaseAddress(pixel_buffer: &cv::PixelBuf) -> *mut u8; + fn CVPixelBufferGetBytesPerRow(pixel_buffer: &cv::PixelBuf) -> usize; + fn CVPixelBufferGetHeight(pixel_buffer: &cv::PixelBuf) -> usize; } -fn pixel_to_cv_format(pixel: Pixel) -> Option { - match pixel { - Pixel::YUYV422 => Some(K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8), - Pixel::UYVY422 => Some(K_CV_PIXEL_FORMAT_TYPE_2VUY), - Pixel::NV12 => Some(K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE), - Pixel::BGRA => Some(K_CV_PIXEL_FORMAT_TYPE_32_BGRA), - Pixel::ARGB => Some(K_CV_PIXEL_FORMAT_TYPE_32_ARGB), - _ => None, - } +fn pixel_to_cv_format(pixel: Pixel) -> Option { + Some(match pixel { + Pixel::YUYV422 => cv::PixelFormat(os::Type::from_be_bytes(*b"yuvs")), + Pixel::UYVY422 => cv::PixelFormat::_2VUY, + Pixel::NV12 => cv::PixelFormat::_420V, + Pixel::BGRA => cv::PixelFormat::_32_BGRA, + Pixel::ARGB => cv::PixelFormat::_32_ARGB, + _ => return None, + }) } -struct SessionHandle(VTPixelTransferSessionRef); +struct SessionHandle(arc::R); unsafe impl Send for SessionHandle {} +unsafe impl Sync for SessionHandle {} + +struct PoolHandle(arc::R); + +unsafe impl Send for PoolHandle {} +unsafe impl Sync for PoolHandle {} pub struct VideoToolboxConverter { session: Mutex, + pool: Option, input_format: Pixel, - input_cv_format: u32, + input_cv_format: cv::PixelFormat, output_format: Pixel, - output_cv_format: u32, + output_cv_format: cv::PixelFormat, input_width: u32, input_height: u32, output_width: u32, @@ -109,6 +53,39 @@ pub struct VideoToolboxConverter { verified_hardware: AtomicBool, } +extern "C" fn noop_release(_: *mut c_void, _: *const *const c_void) {} + +fn create_pool( + width: u32, + height: u32, + format: cv::PixelFormat, +) -> Result { + let width = cf::Number::from_usize(width as usize); + let height = cf::Number::from_usize(height as usize); + let io_surface = cf::Dictionary::new(); + let attrs = cf::DictionaryOf::::with_keys_values( + &[ + cv::pixel_buffer::keys::pixel_format(), + cv::pixel_buffer::keys::width(), + cv::pixel_buffer::keys::height(), + cv::pixel_buffer::keys::io_surf_props(), + ], + &[ + format.as_ref(), + width.as_type_ref(), + height.as_type_ref(), + io_surface.as_type_ref(), + ], + ) + .ok_or_else(|| { + ConvertError::HardwareUnavailable("VideoToolbox pixel buffer pool attributes".to_string()) + })?; + + cv::PixelBufPool::new(None, Some(&**attrs)) + .map(PoolHandle) + .map_err(|error| ConvertError::HardwareUnavailable(format!("CVPixelBufferPool: {error}"))) +} + impl VideoToolboxConverter { pub fn new(config: ConversionConfig) -> Result { let input_cv_format = pixel_to_cv_format(config.input_format).ok_or( @@ -119,20 +96,18 @@ impl VideoToolboxConverter { ConvertError::UnsupportedFormat(config.input_format, config.output_format), )?; - let mut session: VTPixelTransferSessionRef = ptr::null_mut(); - let status = unsafe { VTPixelTransferSessionCreate(ptr::null(), &mut session) }; - - if status != 0 { - return Err(ConvertError::HardwareUnavailable(format!( - "VTPixelTransferSessionCreate failed with status: {status}" - ))); - } + let mut session = vt::PixelTransferSession::new().map_err(|error| { + ConvertError::HardwareUnavailable(format!("VTPixelTransferSessionCreate: {error}")) + })?; + let _ = session.set_realtime(true); - if session.is_null() { - return Err(ConvertError::HardwareUnavailable( - "VTPixelTransferSessionCreate returned null session".to_string(), - )); - } + let pool = match create_pool(config.output_width, config.output_height, output_cv_format) { + Ok(pool) => Some(pool), + Err(error) => { + tracing::debug!("VideoToolbox pixel buffer pool unavailable: {error}"); + None + } + }; tracing::debug!( "VideoToolbox converter initialized: {:?} {}x{} -> {:?} {}x{}", @@ -146,6 +121,7 @@ impl VideoToolboxConverter { Ok(Self { session: Mutex::new(SessionHandle(session)), + pool, input_format: config.input_format, input_cv_format, output_format: config.output_format, @@ -162,119 +138,100 @@ impl VideoToolboxConverter { fn create_input_pixel_buffer( &self, input: &frame::Video, - ) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - + ) -> Result, ConvertError> { let base_address = input.data(0).as_ptr() as *mut c_void; let bytes_per_row = input.stride(0); - let status = unsafe { - CVPixelBufferCreateWithBytes( - ptr::null(), - self.input_width as usize, - self.input_height as usize, - self.input_cv_format, - base_address, - bytes_per_row, - ptr::null(), - ptr::null(), - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreateWithBytes failed: {status}" - ))); - } - - Ok(pixel_buffer) + cv::PixelBuf::with_bytes( + self.input_width as usize, + self.input_height as usize, + base_address, + bytes_per_row, + noop_release, + ptr::null_mut(), + self.input_cv_format, + None, + ) + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferCreateWithBytes: {error}")) + }) } - fn create_output_pixel_buffer(&self) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - - let status = unsafe { - CVPixelBufferCreate( - ptr::null(), - self.output_width as usize, - self.output_height as usize, - self.output_cv_format, - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreate failed: {status}" - ))); + fn create_output_pixel_buffer(&self) -> Result, ConvertError> { + if let Some(pool) = &self.pool + && let Ok(buffer) = pool.0.pixel_buf() + { + return Ok(buffer); } - Ok(pixel_buffer) + cv::PixelBuf::new( + self.output_width as usize, + self.output_height as usize, + self.output_cv_format, + None, + ) + .map_err(|error| ConvertError::ConversionFailed(format!("CVPixelBufferCreate: {error}"))) } fn copy_output_to_frame( &self, - pixel_buffer: CVPixelBufferRef, + pixel_buffer: &mut cv::PixelBuf, output: &mut frame::Video, ) -> Result<(), ConvertError> { - unsafe { - let lock_status = CVPixelBufferLockBaseAddress(pixel_buffer, 0); - if lock_status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferLockBaseAddress failed: {lock_status}" - ))); - } - } + unsafe { pixel_buffer.lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY) } + .result() + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferLockBaseAddress: {error}")) + })?; unsafe { - let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + let plane_count = pixel_buffer.plane_count(); if plane_count == 0 { let src_ptr = CVPixelBufferGetBaseAddress(pixel_buffer); let src_stride = CVPixelBufferGetBytesPerRow(pixel_buffer); let height = CVPixelBufferGetHeight(pixel_buffer); let dst_stride = output.stride(0); - - let dst_data = output.data_mut(0); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(0).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } else { for plane in 0..plane_count { - let src_ptr = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane); - let src_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); - let height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + let src_ptr = pixel_buffer.plane_base_address(plane); + let src_stride = pixel_buffer.plane_bytes_per_row(plane); + let height = pixel_buffer.plane_height(plane); let dst_stride = output.stride(plane); - - let dst_data = output.data_mut(plane); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(plane).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } } + } - CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + unsafe { + let _ = pixel_buffer.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY); } Ok(()) @@ -283,12 +240,9 @@ impl VideoToolboxConverter { impl Drop for VideoToolboxConverter { fn drop(&mut self) { - let session = self.session.get_mut().0; - if !session.is_null() { - unsafe { - VTPixelTransferSessionInvalidate(session); - CFRelease(session as *const c_void); - } + self.session.get_mut().0.invalidate(); + if let Some(pool) = &self.pool { + pool.0.flush(cv::PixelBufPoolFlushFlags::EXCESS_BUFFERS); } } } @@ -317,25 +271,17 @@ impl FrameConverter for VideoToolboxConverter { } let input_buffer = self.create_input_pixel_buffer(&input)?; - let output_buffer = self.create_output_pixel_buffer()?; - - let status = { - let session_guard = self.session.lock(); - unsafe { - VTPixelTransferSessionTransferImage(session_guard.0, input_buffer, output_buffer) - } - }; + let mut output_buffer = self.create_output_pixel_buffer()?; - unsafe { - CVPixelBufferRelease(input_buffer); - } + let status = self + .session + .lock() + .0 + .transfer(&input_buffer, &output_buffer); - if status != 0 { - unsafe { - CVPixelBufferRelease(output_buffer); - } + if let Err(error) = status { return Err(ConvertError::ConversionFailed(format!( - "VTPixelTransferSessionTransferImage failed: {status}" + "VTPixelTransferSessionTransferImage: {error}" ))); } @@ -345,13 +291,9 @@ impl FrameConverter for VideoToolboxConverter { ); } - self.copy_output_to_frame(output_buffer, output)?; + self.copy_output_to_frame(&mut output_buffer, output)?; output.set_pts(input.pts()); - unsafe { - CVPixelBufferRelease(output_buffer); - } - Ok(()) } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index c5f65067c4e..ea6f82b3c2a 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -1265,40 +1265,46 @@ async fn setup_camera( .senders .load(std::sync::atomic::Ordering::Relaxed) > 0 - && let Ok(bytes) = frame.native().bytes() { use cap_mediafoundation_utils::IMFMediaBufferExt; use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; - let data_len = bytes.len(); - if let Ok(buffer) = unsafe { MFCreateMemoryBuffer(data_len as u32) } { - let buffer_ready = { - if let Ok(mut lock) = buffer.lock_for_write() { - lock.copy_from_slice(&bytes); - true - } else { - false - } - }; - - if buffer_ready { - let _ = unsafe { buffer.SetCurrentLength(data_len as u32) }; - - #[allow(clippy::arc_with_non_send_sync)] - let buffer = std::sync::Arc::new(std::sync::Mutex::new(buffer)); - hand_native_frame_to_feed( - &native_recipient, - &native_ingress, - NativeCameraFrame { - buffer, - pixel_format: frame.native().pixel_format, - width: frame.native().width as u32, - height: frame.native().height as u32, - is_bottom_up: frame.native().is_bottom_up, - timestamp, - }, - ); - } + let buffer = if let Some(buffer) = frame.native().media_buffer() { + #[allow(clippy::arc_with_non_send_sync)] + Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + } else if let Ok(bytes) = frame.native().bytes() { + let data_len = bytes.len(); + unsafe { MFCreateMemoryBuffer(data_len as u32) } + .ok() + .and_then(|buffer| { + buffer + .lock_for_write() + .ok() + .map(|mut lock| lock.copy_from_slice(&bytes)) + .map(|()| buffer) + }) + .and_then(|buffer| { + unsafe { buffer.SetCurrentLength(data_len as u32) }.ok()?; + #[allow(clippy::arc_with_non_send_sync)] + Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + }) + } else { + None + }; + + if let Some(buffer) = buffer { + hand_native_frame_to_feed( + &native_recipient, + &native_ingress, + NativeCameraFrame { + buffer, + pixel_format: frame.native().pixel_format, + width: frame.native().width as u32, + height: frame.native().height as u32, + is_bottom_up: frame.native().is_bottom_up, + timestamp, + }, + ); } } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index faab33d6e76..efd9d2341ef 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -506,7 +506,50 @@ impl Muxer for WindowsMuxer { #[cfg(test)] mod tests { - use super::reference_video_frame; + use super::{MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, reference_video_frame}; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_timestamp::Timestamp; + use std::sync::{Arc, Mutex}; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + #[test] + fn native_nv12_frame_converts_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 13) as u8; + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let base_ptr = { + let lock = buffer.lock().expect("read lock"); + lock.as_ptr() + }; + + #[allow(clippy::arc_with_non_send_sync)] + let frame = NativeCameraFrame { + buffer: Arc::new(Mutex::new(buffer)), + pixel_format: PixelFormat::NV12, + width: width as u32, + height: height as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(std::time::Instant::now()), + }; + let mut decoder = MjpegDecoder::default(); + let converted = camera_frame_to_ffmpeg(&frame, &mut decoder).expect("conversion"); + + assert_eq!(converted.format(), ffmpeg::format::Pixel::NV12); + assert_eq!(converted.data(0).as_ptr(), base_ptr); + } #[test] fn retained_software_frame_shares_pixels_and_survives_original() { @@ -1357,6 +1400,21 @@ pub(crate) fn camera_frame_to_ffmpeg( return decode_mjpeg_frame(frame, mjpeg_decoder); } + if frame.pixel_format != cap_camera_windows::PixelFormat::UYVY422 + && frame.pixel_format != cap_camera_windows::PixelFormat::NV21 + && let Ok(buffer) = frame.buffer.lock() + && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( + &buffer, + frame.pixel_format, + frame.width as usize, + frame.height as usize, + frame.is_bottom_up, + ) + { + cap_camera_ffmpeg::CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + let ffmpeg_format = match frame.pixel_format { cap_camera_windows::PixelFormat::NV12 => ffmpeg::format::Pixel::NV12, cap_camera_windows::PixelFormat::YUYV422 => ffmpeg::format::Pixel::YUYV422, From 603acb9f5968383744d2344dc5814e8a7633cef8 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 00:49:30 +0100 Subject: [PATCH 2/7] fix(camera): align capture lock paths and preserve preview conversions Use the same Media Foundation lock type for every consumer of a capture buffer, including the hardware upload and preview copy paths, and cache the last converted frame for paced repeats. Revert the macOS camera-ffmpeg zero-copy wrapper so preview and ffmpeg consumers copy instead of holding CVPixelBuffer locks for an AVFrame lifetime. Fix the VideoToolbox pool attribute compile error, share DirectShow media types across the callback worker, initialize that worker as MTA, and expose callback drop counters. Restore NV21 to NV12 conversion in as_ffmpeg and log conversion statistics periodically. --- crates/camera-directshow/src/lib.rs | 76 +++++-- .../examples/frame-copy-bench.rs | 17 +- crates/camera-ffmpeg/src/lib.rs | 15 ++ crates/camera-ffmpeg/src/macos.rs | 199 +---------------- crates/camera-ffmpeg/src/windows.rs | 203 ++++++++++++++++-- crates/camera-mediafoundation/src/lib.rs | 8 +- crates/camera-windows/src/lib.rs | 7 + crates/camera/src/lib.rs | 28 +++ crates/camera/src/windows.rs | 4 + crates/frame-converter/src/videotoolbox.rs | 5 +- crates/recording/src/output_pipeline/win.rs | 63 ++++-- .../src/output_pipeline/win_fragmented_m4s.rs | 140 +++++++----- 12 files changed, 450 insertions(+), 315 deletions(-) diff --git a/crates/camera-directshow/src/lib.rs b/crates/camera-directshow/src/lib.rs index 3fb74fa08ad..81043f845de 100644 --- a/crates/camera-directshow/src/lib.rs +++ b/crates/camera-directshow/src/lib.rs @@ -39,6 +39,10 @@ pub fn initialize_directshow() -> windows_core::Result<()> { unsafe { CoInitialize(None) }.ok() } +fn initialize_directshow_worker() { + let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; +} + pub trait IPinExt { /// # Safety /// Do it correctly @@ -465,8 +469,10 @@ impl VideoInputDevice { .SetFormat(&**format) .map_err(StartCapturingError::Other)?; - let (sink_filter, callback_worker) = SinkFilter::new(format.clone(), callback) - .map_err(StartCapturingError::CallbackWorker)?; + let dropped = Arc::new(AtomicU64::new(0)); + let (sink_filter, callback_worker) = + SinkFilter::new(format.clone(), callback, dropped.clone()) + .map_err(StartCapturingError::CallbackWorker)?; let input_sink_pin = sink_filter .get_pin(0) @@ -522,6 +528,7 @@ impl VideoInputDevice { output_capture_pin: bound.output_pin, input_sink_pin, callback_worker, + dropped_samples: dropped, }) } } @@ -533,9 +540,14 @@ pub struct CaptureHandle { output_capture_pin: IPin, input_sink_pin: IPin, callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + // Chromium: VideoCaptureDeviceWin::StopAndDeallocate pub fn stop_capturing(self) -> windows_core::Result<()> { unsafe { self.media_control.Stop() }?; @@ -594,6 +606,11 @@ impl Iterator for VideoMediaTypesIterator<'_> { #[derive(Default, Debug)] pub struct AMMediaType(AM_MEDIA_TYPE); +// The media type owns its format block and is immutable after construction; +// the callback worker only reads it. +unsafe impl Send for AMMediaType {} +unsafe impl Sync for AMMediaType {} + impl AMMediaType { pub fn new(typ: &AM_MEDIA_TYPE) -> Self { Self(unsafe { copy_media_type(typ) }) @@ -605,6 +622,32 @@ impl AMMediaType { std::mem::forget(wrapper); inner } + + fn matches_media_type(&self, other: &AM_MEDIA_TYPE) -> bool { + let a = &self.0; + let b = other; + + if a.majortype != b.majortype + || a.subtype != b.subtype + || a.formattype != b.formattype + || a.cbFormat != b.cbFormat + { + return false; + } + + if a.cbFormat == 0 { + return true; + } + + if a.pbFormat.is_null() || b.pbFormat.is_null() { + return a.pbFormat == b.pbFormat; + } + + unsafe { + std::slice::from_raw_parts(a.pbFormat, a.cbFormat as usize) + == std::slice::from_raw_parts(b.pbFormat, b.cbFormat as usize) + } + } } impl Deref for AMMediaType { @@ -643,9 +686,9 @@ impl SinkFilter { pub(crate) fn new( desired_media_type: AMMediaType, callback: SinkCallback, + dropped: Arc, ) -> std::io::Result<(ComObject, CallbackWorker)> { let (callback_tx, callback_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); - let dropped = Arc::new(AtomicU64::new(0)); let callback_worker = CallbackWorker::spawn(callback, callback_rx, Arc::new(AtomicBool::new(false)))?; @@ -841,7 +884,7 @@ const CALLBACK_QUEUE_DEPTH: usize = 4; struct OwnedCallbackData { sample: IMediaSample, - media_type: AMMediaType, + media_type: Arc, timestamp: Duration, perf_counter: i64, } @@ -863,12 +906,12 @@ impl CallbackWorker { let handle = thread::Builder::new() .name("dshow-camera-callback".to_string()) .spawn(move || { - let _ = initialize_directshow(); + initialize_directshow_worker(); while !worker_stop.load(Ordering::Acquire) { match samples.recv_timeout(Duration::from_millis(100)) { Ok(data) => callback(CallbackData { sample: &data.sample, - media_type: &data.media_type, + media_type: data.media_type.as_ref(), timestamp: data.timestamp, perf_counter: data.perf_counter, }), @@ -897,8 +940,7 @@ impl Drop for CallbackWorker { #[implement(IPin, IMemInputPin)] struct SinkInputPin { desired_media_type: AMMediaType, - #[allow(unused)] - current_media_type: RefCell, + current_media_type: RefCell>, connected_pin: RefCell>, owner: RefCell>, callback_tx: SyncSender, @@ -969,7 +1011,7 @@ impl IPin_Impl for SinkInputPin_Impl { }; *connected_pin = Some(pconnector.clone()); self.current_media_type - .replace(AMMediaType::new(unsafe { &*pmt })); + .replace(Arc::new(AMMediaType::new(unsafe { &*pmt }))); S_OK.ok() } @@ -996,7 +1038,7 @@ impl IPin_Impl for SinkInputPin_Impl { .as_ref() .ok_or(VFW_E_NOT_CONNECTED)?; - unsafe { *pmt = self.current_media_type.borrow().clone().into_inner() }; + unsafe { *pmt = (**self.current_media_type.borrow()).clone().into_inner() }; Ok(()) } @@ -1099,16 +1141,16 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { return Ok(()); }; - unsafe { - if let Ok(new_media_type) = psample.GetMediaType() + let media_type = { + let mut current = self.current_media_type.borrow_mut(); + if let Ok(new_media_type) = unsafe { psample.GetMediaType() } && !new_media_type.is_null() + && !current.matches_media_type(unsafe { &*new_media_type }) { - self.current_media_type - .replace(AMMediaType::new(&*new_media_type)); + *current = Arc::new(AMMediaType::new(unsafe { &*new_media_type })); } - } - - let media_type = self.current_media_type.borrow().clone(); + Arc::clone(&*current) + }; let length = unsafe { psample.GetActualDataLength() }; diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs index 9e9fe178b0d..cf090310cef 100644 --- a/crates/camera-ffmpeg/examples/frame-copy-bench.rs +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -1,7 +1,7 @@ #[cfg(windows)] fn main() { use cap_camera::CapturedFrame; - use cap_camera_ffmpeg::{CapturedFrameExt, conversion_stats}; + use cap_camera_ffmpeg::media_buffer_as_ffmpeg; use cap_camera_windows::{Frame, PixelFormat}; use cap_mediafoundation_utils::IMFMediaBufferExt; use ffmpeg::{format::Pixel, frame::Video}; @@ -33,14 +33,18 @@ fn main() { ); let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); + let native_buffer = captured.native().media_buffer().expect("native buffer"); + for _ in 0..50 { - let frame = captured.as_ffmpeg().expect("warmup conversion"); + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("warmup conversion"); std::hint::black_box(frame); } let start = Instant::now(); for _ in 0..ITERATIONS { - let frame = captured.as_ffmpeg().expect("conversion"); + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("conversion"); std::hint::black_box(frame); } let zero_copy = start.elapsed(); @@ -78,7 +82,6 @@ fn main() { } let bytes_copy = start.elapsed(); - let stats = conversion_stats(); println!( "zero-copy: {:.3} ms/frame ({:.1} fps)", zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, @@ -97,12 +100,6 @@ fn main() { "bytes copy: {:.3} ms/frame", bytes_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64 ); - println!( - "stats: zero_copy={} copied={} avg_copy={:.3} us", - stats.zero_copy_frames, - stats.copied_frames, - stats.average_copy_micros() - ); } #[cfg(not(windows))] diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index c8cbcbb1a6d..d9cdd616c3d 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -45,11 +45,26 @@ pub struct ConversionStats { impl ConversionStats { pub fn record_zero_copy(&self) { self.zero_copy_frames.fetch_add(1, Ordering::Relaxed); + self.log_periodically(); } pub fn record_copy(&self, elapsed_nanos: u64) { self.copied_frames.fetch_add(1, Ordering::Relaxed); self.copy_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); + self.log_periodically(); + } + + fn log_periodically(&self) { + let snapshot = self.snapshot(); + let total = snapshot.zero_copy_frames + snapshot.copied_frames; + if total > 0 && total.is_multiple_of(300) { + tracing::debug!( + zero_copy_frames = snapshot.zero_copy_frames, + copied_frames = snapshot.copied_frames, + average_copy_micros = snapshot.average_copy_micros(), + "Camera frame conversion stats" + ); + } } pub fn snapshot(&self) -> ConversionStatsSnapshot { diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index b9892dbb74b..fa27b56a4a3 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -1,29 +1,14 @@ use cap_camera::CapturedFrame; use cap_camera_avfoundation::ImageBufExt; use cidre::*; -use ffmpeg::{ffi, format::Pixel, software::scaling}; +use ffmpeg::{format::Pixel, software::scaling}; use std::{ cell::RefCell, - ffi::c_void, - ptr, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, + sync::atomic::{AtomicBool, Ordering}, }; use crate::{CONVERSION_STATS, CapturedFrameExt}; -type CVPixelBufferRef = *mut c_void; - -#[link(name = "CoreVideo", kind = "framework")] -unsafe extern "C" { - fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; - fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetWidth(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; -} - type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video); #[derive(thiserror::Error, Debug)] @@ -73,181 +58,6 @@ fn fourcc_to_pixel_format(fourcc: &str) -> Option { static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false); -#[link(name = "CoreVideo", kind = "framework")] -unsafe extern "C" { - fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; -} - -const CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 1; - -#[derive(Clone, Copy)] -struct SamplePlane { - ptr: *mut u8, - len: usize, - stride: usize, -} - -struct WrappedSampleBuf { - _image_buf: cidre::arc::R, - pixel_buffer: CVPixelBufferRef, - locked: bool, -} - -unsafe impl Send for WrappedSampleBuf {} -unsafe impl Sync for WrappedSampleBuf {} - -impl Drop for WrappedSampleBuf { - fn drop(&mut self) { - if self.locked { - unsafe { - CVPixelBufferUnlockBaseAddress(self.pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); - } - } - } -} - -unsafe extern "C" fn release_wrapped_sample_buf(opaque: *mut c_void, _data: *mut u8) { - if !opaque.is_null() { - drop(unsafe { Box::from_raw(opaque as *mut Arc) }); - } -} - -fn sample_buf_planes( - image_buf: &cidre::cv::ImageBuf, - format: &str, -) -> Option<(Pixel, Vec, Option)> { - let (pixel, plane_count, range) = match format { - "2vuy" => (Pixel::UYVY422, 0, None), - "yuvs" => (Pixel::YUYV422, 0, None), - "BGRA" => (Pixel::BGRA, 0, None), - "ARGB" => (Pixel::ARGB, 0, None), - "24BG" => (Pixel::BGR24, 0, None), - "24RG" => (Pixel::RGB24, 0, None), - "L008" | "GRAY" => (Pixel::GRAY8, 0, None), - "L016" => (Pixel::GRAY16LE, 0, None), - "420v" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::MPEG)), - "420f" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::JPEG)), - "y420" => (Pixel::YUV420P, 3, None), - _ => return None, - }; - - if image_buf.plane_count() != plane_count { - return None; - } - - let mut planes = Vec::with_capacity(plane_count.max(1)); - if plane_count == 0 { - let pixel_buffer = image_buf as *const cidre::cv::ImageBuf as CVPixelBufferRef; - let ptr = unsafe { CVPixelBufferGetBaseAddress(pixel_buffer) }; - let stride = unsafe { CVPixelBufferGetBytesPerRow(pixel_buffer) }; - let height = unsafe { CVPixelBufferGetHeight(pixel_buffer) }; - if ptr.is_null() || stride == 0 || height == 0 { - return None; - } - planes.push(SamplePlane { - ptr, - len: stride.checked_mul(height)?, - stride, - }); - } else { - for plane in 0..plane_count { - let ptr = image_buf.plane_base_address(plane) as *mut u8; - let stride = image_buf.plane_bytes_per_row(plane); - let height = image_buf.plane_height(plane); - if ptr.is_null() || stride == 0 || height == 0 { - return None; - } - planes.push(SamplePlane { - ptr, - len: stride.checked_mul(height)?, - stride, - }); - } - } - - Some((pixel, planes, range)) -} - -fn wrap_sample_buf(sample_buf: &cidre::cm::SampleBuf) -> Option { - let image_buf = sample_buf.image_buf()?.retained(); - let format_desc = sample_buf.format_desc()?; - let format = cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()); - let pixel_buffer = (&*image_buf) as *const cidre::cv::ImageBuf as CVPixelBufferRef; - - if unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY) } != 0 { - return None; - } - - let Some((pixel, planes, range)) = sample_buf_planes(&image_buf, format) else { - unsafe { - CVPixelBufferUnlockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); - } - return None; - }; - - let owner = Arc::new(WrappedSampleBuf { - _image_buf: image_buf, - pixel_buffer, - locked: true, - }); - - let mut frame = ffmpeg::frame::Video::empty(); - let ptr = unsafe { frame.as_mut_ptr() }; - let format: ffi::AVPixelFormat = pixel.into(); - unsafe { - (*ptr).format = format as i32; - (*ptr).width = image_buf_width(&owner) as i32; - (*ptr).height = image_buf_height(&owner) as i32; - } - - let mut buffers = Vec::with_capacity(planes.len()); - for plane in &planes { - let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; - let buffer = unsafe { - ffi::av_buffer_create( - plane.ptr, - plane.len, - Some(release_wrapped_sample_buf), - opaque, - ffi::AV_BUFFER_FLAG_READONLY, - ) - }; - if buffer.is_null() { - unsafe { - drop(Box::from_raw(opaque as *mut Arc)); - for mut buffer in buffers { - ffi::av_buffer_unref(&mut buffer); - } - } - return None; - } - buffers.push(buffer); - } - - for (index, (buffer, plane)) in buffers.into_iter().zip(&planes).enumerate() { - unsafe { - (*ptr).buf[index] = buffer; - (*ptr).data[index] = plane.ptr; - (*ptr).linesize[index] = plane.stride as i32; - } - } - - if let Some(range) = range { - frame.set_color_range(range); - } - - Some(frame) -} - -fn image_buf_width(owner: &WrappedSampleBuf) -> usize { - unsafe { CVPixelBufferGetWidth(owner.pixel_buffer) } -} - -fn image_buf_height(owner: &WrappedSampleBuf) -> usize { - unsafe { CVPixelBufferGetHeight(owner.pixel_buffer) } -} - impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) @@ -267,11 +77,6 @@ impl CapturedFrameExt for CapturedFrame { pub fn sample_buf_as_ffmpeg( sample_buf: &cm::SampleBuf, ) -> Result { - if let Some(frame) = wrap_sample_buf(sample_buf) { - CONVERSION_STATS.record_zero_copy(); - return Ok(frame); - } - let started = std::time::Instant::now(); let frame = { let mut image_buf = sample_buf diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 92a7326beb4..9873f1cc878 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -372,9 +372,7 @@ fn lock_mf_buffer( format: PixelFormat, width: usize, height: usize, -) -> Option<(WrappedMfBuffer, Vec, Pixel)> { - let pixel = mf_pixel_format(format)?; - +) -> Option<(WrappedMfBuffer, Vec)> { unsafe { if let Ok(buffer_2d) = buffer.cast::() { let mut scanline = ptr::null_mut(); @@ -403,7 +401,6 @@ fn lock_mf_buffer( lock: MfLock::TwoD2(buffer_2d), }, planes, - pixel, )); } @@ -433,7 +430,6 @@ fn lock_mf_buffer( lock: MfLock::TwoD(buffer_2d), }, planes, - pixel, )); } @@ -456,7 +452,6 @@ fn lock_mf_buffer( lock: MfLock::Contiguous, }, planes, - pixel, )); } @@ -474,14 +469,77 @@ pub fn media_buffer_as_ffmpeg( height: usize, is_bottom_up: bool, ) -> Option { - if is_bottom_up || width == 0 || height == 0 { + if !should_use_2d_lock(format, is_bottom_up) || width == 0 || height == 0 { return None; } - let (owner, planes, pixel) = lock_mf_buffer(buffer, format, width, height)?; + let pixel = mf_pixel_format(format)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) } +// UYVY and NV21 use the format-converting copy paths, which lock with +// IMFMediaBuffer::Lock; all consumers of a buffer must use the same lock type. +pub fn should_use_2d_lock(format: PixelFormat, is_bottom_up: bool) -> bool { + !is_bottom_up + && format != PixelFormat::UYVY422 + && format != PixelFormat::NV21 + && mf_pixel_format(format).is_some() +} + +pub fn with_media_buffer_data( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + f: impl FnOnce(&[u8], usize, usize) -> R, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let data = unsafe { std::slice::from_raw_parts(planes[0].ptr, total_len) }; + let primary_stride = planes[0].stride; + let secondary_stride = planes.get(1).map_or(primary_stride, |plane| plane.stride); + let result = f(data, primary_stride, secondary_stride); + drop(owner); + Some(result) +} + +pub fn copy_media_buffer_to_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let pixel = mf_pixel_format(format)?; + let row_bytes = mf_row_bytes(format, width)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let mut frame = FFVideo::new(pixel, width as u32, height as u32); + + for (index, plane) in planes.iter().enumerate() { + let plane_height = plane.len / plane.stride; + let destination_stride = frame.stride(index); + let destination = frame.data_mut(index); + + for row in 0..plane_height { + let source = + unsafe { std::slice::from_raw_parts(plane.ptr.add(row * plane.stride), row_bytes) }; + let destination_start = row * destination_stride; + destination[destination_start..destination_start + row_bytes].copy_from_slice(source); + } + } + + drop(owner); + Some(frame) +} + fn wrap_mf_frame( planes: &[MfPlane], pixel: Pixel, @@ -554,11 +612,14 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } - if let Some(buffer) = native.media_buffer() + let started = std::time::Instant::now(); + + if should_use_2d_lock(native.pixel_format, is_bottom_up) + && let Some(buffer) = native.media_buffer() && let Some(frame) = - media_buffer_as_ffmpeg(&buffer, native.pixel_format, width, height, is_bottom_up) + copy_media_buffer_to_ffmpeg(&buffer, native.pixel_format, width, height) { - CONVERSION_STATS.record_zero_copy(); + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); return Ok(frame); } @@ -568,7 +629,6 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } - let started = std::time::Instant::now(); let frame = match native.pixel_format { PixelFormat::YUV420P => { let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32); @@ -833,10 +893,96 @@ impl CapturedFrameExt for CapturedFrame { } #[cfg(test)] -mod zero_copy_tests { +mod conversion_tests { use super::*; + use cap_camera_windows::Frame; use cap_mediafoundation_utils::IMFMediaBufferExt; - use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; + + #[test] + fn nv12_2d_buffer_can_be_wrapped_twice() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let first = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("first frame"); + let second = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + + assert!(second.is_some(), "second lock failed"); + drop(first); + let third = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + assert!(third.is_some(), "third lock failed"); + } + + #[test] + fn copy_conversion_can_nest_inside_zero_copy_frame() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let zero_copy = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("zero-copy frame"); + let copied = copy_media_buffer_to_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + ) + .expect("copied frame"); + + assert_eq!(copied.format(), Pixel::NV12); + assert_eq!(copied.width(), width); + assert_eq!(copied.height(), height); + + let data_len = with_media_buffer_data( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + |data, primary_stride, secondary_stride| { + assert_eq!(primary_stride, secondary_stride); + assert!(primary_stride >= width as usize); + data.len() + }, + ) + .expect("locked data"); + + assert!(data_len >= width as usize * height as usize * 3 / 2); + drop(zero_copy); + } #[test] fn nv12_media_buffer_is_wrapped_without_copying() { @@ -877,4 +1023,33 @@ mod zero_copy_tests { assert_eq!(frame.data(1)[0], 0); assert_eq!(frame.data(1)[stride + 2], 5); } + + #[test] + fn as_ffmpeg_keeps_nv12_for_nv21_frames() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = Frame::from_media_foundation( + buffer, + PixelFormat::NV21, + width, + height, + false, + std::time::Duration::ZERO, + 0, + ); + let captured = + cap_camera::CapturedFrame::from_windows_frame(frame, std::time::Duration::ZERO); + let converted = captured.as_ffmpeg().expect("conversion"); + + assert_eq!(converted.format(), Pixel::NV12); + assert_eq!(converted.width(), width as u32); + assert_eq!(converted.height(), height as u32); + } } diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index e09fa426e21..4378ef24b5c 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -264,7 +264,7 @@ impl Device { let video_callback = VideoCallback { event_tx, samples: sample_tx, - dropped, + dropped: dropped.clone(), } .into_object(); @@ -389,6 +389,7 @@ impl Device { engine: Some(engine), event_rx, callback_worker, + dropped_samples: dropped, }) } } @@ -425,6 +426,7 @@ pub struct CaptureHandle { engine: Option, event_rx: Receiver, callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { @@ -432,6 +434,10 @@ impl CaptureHandle { &self.event_rx } + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + pub fn stop_capturing(mut self) -> windows_core::Result<()> { let result = if let Some(engine) = self.engine.take() { unsafe { engine.StopPreview() } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index 526a279ffd6..c1badbd9e68 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -173,6 +173,13 @@ pub enum CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + match self { + CaptureHandle::MediaFoundation(handle) => handle.dropped_samples(), + CaptureHandle::DirectShow(handle) => handle.dropped_samples(), + } + } + pub fn stop_capturing(self) -> windows_core::Result<()> { match self { CaptureHandle::MediaFoundation(handle) => handle.stop_capturing(), diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index 74f64a642eb..669b0ac794d 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -322,7 +322,28 @@ pub struct CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + #[cfg(windows)] + { + self.native + .as_ref() + .map(NativeCaptureHandle::dropped_samples) + .unwrap_or(0) + } + #[cfg(not(windows))] + { + 0 + } + } + pub fn stop_capturing(mut self) -> Result<(), String> { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture stopped with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing()?; } @@ -332,6 +353,13 @@ impl CaptureHandle { impl Drop for CaptureHandle { fn drop(&mut self) { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture released with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing().ok(); } diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index 07d72fa184d..73c9264cec2 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -96,6 +96,10 @@ pub struct WindowsCaptureHandle { } impl WindowsCaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.inner.dropped_samples() + } + pub fn stop_capturing(self) -> Result<(), String> { self.inner.stop_capturing().map_err(|e| e.to_string()) } diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index e0a104cd95e..b16a3b65f42 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -76,10 +76,7 @@ fn create_pool( height.as_type_ref(), io_surface.as_type_ref(), ], - ) - .ok_or_else(|| { - ConvertError::HardwareUnavailable("VideoToolbox pixel buffer pool attributes".to_string()) - })?; + ); cv::PixelBufPool::new(None, Some(&**attrs)) .map(PoolHandle) diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index efd9d2341ef..c567a82e94a 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -1400,8 +1400,7 @@ pub(crate) fn camera_frame_to_ffmpeg( return decode_mjpeg_frame(frame, mjpeg_decoder); } - if frame.pixel_format != cap_camera_windows::PixelFormat::UYVY422 - && frame.pixel_format != cap_camera_windows::PixelFormat::NV21 + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) && let Ok(buffer) = frame.buffer.lock() && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( &buffer, @@ -1867,25 +1866,45 @@ pub fn upload_mf_buffer_to_texture( buffers: &mut CameraBuffers, ) -> windows::core::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - use windows::Win32::Graphics::Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, - }; - use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; - - let dxgi_format = frame.dxgi_format(); let buffer_guard = frame .buffer .lock() .map_err(|_| windows::core::Error::from(windows::core::HRESULT(-1)))?; - let lock = buffer_guard.lock()?; - let original_data = &*lock; + let buffer = &*buffer_guard; + + // The zero-copy and preview paths lock these formats with Lock2D; the + // upload must use the same lock type to avoid MF_E_INVALIDREQUEST. + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) + && let Some(texture) = cap_camera_ffmpeg::with_media_buffer_data( + buffer, + frame.pixel_format, + frame.width as usize, + frame.height as usize, + |data, primary_stride, secondary_stride| { + upload_mf_buffer_data_to_texture( + device, + frame, + data, + CameraBufferLayout { + primary_stride, + secondary_stride, + }, + buffers, + ) + }, + ) + { + return texture; + } + + let lock = buffer.lock()?; + let data = &*lock; let layout = infer_camera_buffer_layout( frame.pixel_format, frame.width as usize, frame.height as usize, - original_data.len(), + data.len(), ) .ok_or_else(|| { windows::core::Error::new( @@ -1895,11 +1914,29 @@ pub fn upload_mf_buffer_to_texture( frame.pixel_format, frame.width, frame.height, - original_data.len() + data.len() ), ) })?; + upload_mf_buffer_data_to_texture(device, frame, data, layout, buffers) +} + +fn upload_mf_buffer_data_to_texture( + device: &ID3D11Device, + frame: &NativeCameraFrame, + original_data: &[u8], + layout: CameraBufferLayout, + buffers: &mut CameraBuffers, +) -> windows::core::Result { + use windows::Win32::Graphics::Direct3D11::{ + D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, + }; + use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; + + let dxgi_format = frame.dxgi_format(); + let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; let needs_flip = frame.is_bottom_up; let mut row_pitch = layout.primary_stride as u32; diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index f14355fc391..d095a6c1076 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -860,7 +860,9 @@ impl WindowsFragmentedM4SCameraMuxer { } let mut pacer = FramePacer::new(video_config.fps()); - let mut last_frame: Option = None; + // Repeats share the last converted frame so the capture buffer + // is locked once instead of on every paced repeat. + let mut last_ffmpeg_frame: Option = None; let mut first_timestamp: Option = None; let mut slow_convert_count = 0u32; @@ -883,17 +885,14 @@ impl WindowsFragmentedM4SCameraMuxer { } }; - let encode_camera_frame_fn = + let convert_camera_frame = |camera_frame: &NativeCameraFrame, - timestamp: Duration, slow_convert_count: &mut u32, - slow_encode_count: &mut u32, - total_frames: &mut u64, - encoder: &Arc>, mjpeg_decoder: &mut MjpegDecoder| - -> anyhow::Result<()> { + -> anyhow::Result { let convert_start = std::time::Instant::now(); - let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); + let ffmpeg_frame_result = + camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); let convert_elapsed_ms = convert_start.elapsed().as_millis(); if convert_elapsed_ms > SLOW_THRESHOLD_MS { @@ -908,46 +907,47 @@ impl WindowsFragmentedM4SCameraMuxer { } } - match ffmpeg_frame_result { - Ok(ffmpeg_frame) => { - let encode_start = std::time::Instant::now(); - - match encoder.lock() { - Ok(mut enc) => { - let result = enc.queue_frame(ffmpeg_frame, timestamp); - report_encode_result( - &mut enc, - result, - &encode_failures, - &health_tx, - "camera", - ); - } - Err(_) => { - return Err(anyhow!( - "Camera encoder mutex poisoned - all subsequent frames would be lost" - )); - } - } + ffmpeg_frame_result + }; - let encode_elapsed_ms = encode_start.elapsed().as_millis(); - - if encode_elapsed_ms > SLOW_THRESHOLD_MS { - *slow_encode_count += 1; - if *slow_encode_count <= 5 - || slow_encode_count.is_multiple_of(100) - { - debug!( - elapsed_ms = encode_elapsed_ms, - count = *slow_encode_count, - "Camera encoder.queue_frame exceeded {}ms threshold", - SLOW_THRESHOLD_MS - ); - } - } + let encode_camera_frame_fn = + |ffmpeg_frame: ffmpeg::frame::Video, + timestamp: Duration, + slow_encode_count: &mut u32, + total_frames: &mut u64, + encoder: &Arc>| + -> anyhow::Result<()> { + let encode_start = std::time::Instant::now(); + + match encoder.lock() { + Ok(mut enc) => { + let result = enc.queue_frame(ffmpeg_frame, timestamp); + report_encode_result( + &mut enc, + result, + &encode_failures, + &health_tx, + "camera", + ); } - Err(e) => { - warn!("Failed to convert camera frame: {e:?}"); + Err(_) => { + return Err(anyhow!( + "Camera encoder mutex poisoned - all subsequent frames would be lost" + )); + } + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + *slow_encode_count += 1; + if *slow_encode_count <= 5 || slow_encode_count.is_multiple_of(100) { + debug!( + elapsed_ms = encode_elapsed_ms, + count = *slow_encode_count, + "Camera encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); } } @@ -967,17 +967,30 @@ impl WindowsFragmentedM4SCameraMuxer { break; } - let (camera_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { - Paced::Frame(frame, ts) => { - last_frame = Some(frame.clone()); - (frame, ts) - } + let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, ts) => match convert_camera_frame( + &frame, + &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => { + last_ffmpeg_frame = Some(reference_video_frame(&ffmpeg_frame)); + (ffmpeg_frame, ts) + } + Err(e) => { + warn!("Failed to convert camera frame: {e:?}"); + let Some(last) = last_ffmpeg_frame.as_ref() else { + continue; + }; + (reference_video_frame(last), ts) + } + }, Paced::Repeat(ts) => { - let Some(frame) = &last_frame else { + let Some(last) = last_ffmpeg_frame.as_ref() else { continue; }; duplicated_frames += 1; - (frame.clone(), ts) + (reference_video_frame(last), ts) } Paced::End => { let mut drained = 0u64; @@ -989,14 +1002,25 @@ impl WindowsFragmentedM4SCameraMuxer { Ok(Some((frame, ts))) => { let normalized_ts = normalize_timestamp(ts, &mut first_timestamp); - if let Err(e) = encode_camera_frame_fn( + let ffmpeg_frame = match convert_camera_frame( &frame, - normalized_ts, &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => ffmpeg_frame, + Err(e) => { + warn!( + "Failed to convert drained camera frame: {e:?}" + ); + continue; + } + }; + if let Err(e) = encode_camera_frame_fn( + ffmpeg_frame, + normalized_ts, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { warn!("Failed to encode drained camera frame: {e}"); break; @@ -1025,13 +1049,11 @@ impl WindowsFragmentedM4SCameraMuxer { let normalized_ts = normalize_timestamp(timestamp, &mut first_timestamp); if let Err(e) = encode_camera_frame_fn( - &camera_frame, + ffmpeg_frame, normalized_ts, - &mut slow_convert_count, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { error!("Camera encoder mutex poisoned - encoder thread likely panicked, stopping"); return Err(e); From 76e90e4b2f5d33f4e3d9e639711f3e749cc8819c Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 05:41:46 +0100 Subject: [PATCH 3/7] perf(camera): remove per-frame allocations and GPU copies on Windows Wrap each MF buffer in a single AVBufferRef with stack plane metadata, cutting the zero-copy path from 0.002 to 0.001 ms/frame. Replace the per-frame Arc> with a direct COM-backed CameraBuffer. Cache the D3D11 upload texture and immediate context and use UpdateSubresource instead of CreateTexture2D per frame: 0.733 vs 0.956 ms/frame for 1080p NV12. Upload straight into the hardware encoder's video-processor input texture to skip the extra CopyResource, while still running VideoProcessorBlt for conversion and scaling. Add camera-upload and zero-copy/preview-copy benchmarks. --- .../examples/frame-copy-bench.rs | 77 ++++++-- crates/camera-ffmpeg/src/windows.rs | 121 ++++++------ crates/enc-mediafoundation/src/video/h264.rs | 11 +- .../src/video/video_processor.rs | 27 ++- .../recording/examples/camera-upload-bench.rs | 130 +++++++++++++ crates/recording/src/feeds/camera.rs | 8 +- crates/recording/src/lib.rs | 2 + crates/recording/src/output_pipeline/win.rs | 181 ++++++++++++------ crates/verify/tests/recording.rs | 6 +- 9 files changed, 418 insertions(+), 145 deletions(-) create mode 100644 crates/recording/examples/camera-upload-bench.rs diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs index cf090310cef..ef56f6f12da 100644 --- a/crates/camera-ffmpeg/examples/frame-copy-bench.rs +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -1,29 +1,36 @@ #[cfg(windows)] fn main() { use cap_camera::CapturedFrame; - use cap_camera_ffmpeg::media_buffer_as_ffmpeg; + use cap_camera_ffmpeg::{copy_media_buffer_to_ffmpeg, media_buffer_as_ffmpeg}; use cap_camera_windows::{Frame, PixelFormat}; use cap_mediafoundation_utils::IMFMediaBufferExt; use ffmpeg::{format::Pixel, frame::Video}; use std::time::{Duration, Instant}; - use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; const WIDTH: usize = 1920; const HEIGHT: usize = 1080; const ITERATIONS: usize = 1000; let len = WIDTH * HEIGHT * 3 / 2; - let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + + let surface = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + } + .expect("2D camera buffer"); { - let mut lock = buffer.lock_for_write().expect("buffer lock"); + let mut lock = surface.lock_for_write().expect("surface lock"); for (index, byte) in lock.iter_mut().enumerate() { *byte = (index * 31) as u8; } } - unsafe { buffer.SetCurrentLength(len as u32) }.expect("buffer length"); - - let frame = Frame::from_media_foundation( - buffer, + let surface_frame = Frame::from_media_foundation( + surface, PixelFormat::NV12, WIDTH, HEIGHT, @@ -31,10 +38,29 @@ fn main() { Duration::ZERO, 0, ); - let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); - + let captured = CapturedFrame::from_windows_frame(surface_frame, Duration::ZERO); let native_buffer = captured.native().media_buffer().expect("native buffer"); + let memory = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + { + let mut lock = memory.lock_for_write().expect("buffer lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + unsafe { memory.SetCurrentLength(len as u32) }.expect("buffer length"); + let memory_frame = Frame::from_media_foundation( + memory, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let copy_captured = CapturedFrame::from_windows_frame(memory_frame, Duration::ZERO); + let copy_bytes = copy_captured.native().bytes().expect("camera bytes"); + for _ in 0..50 { let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) .expect("warmup conversion"); @@ -49,20 +75,34 @@ fn main() { } let zero_copy = start.elapsed(); - let bytes = captured.native().bytes().expect("camera bytes"); + for _ in 0..50 { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("warmup preview copy"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("preview copy"); + std::hint::black_box(frame); + } + let preview_copy = start.elapsed(); + let start = Instant::now(); for _ in 0..ITERATIONS { let mut frame = Video::new(Pixel::NV12, WIDTH as u32, HEIGHT as u32); let stride = frame.stride(0); for row in 0..HEIGHT { frame.data_mut(0)[row * stride..row * stride + WIDTH] - .copy_from_slice(&bytes[row * WIDTH..row * WIDTH + WIDTH]); + .copy_from_slice(©_bytes[row * WIDTH..row * WIDTH + WIDTH]); } let uv_stride = frame.stride(1); let uv_offset = WIDTH * HEIGHT; for row in 0..HEIGHT / 2 { - frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH] - .copy_from_slice(&bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH]); + frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH].copy_from_slice( + ©_bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH], + ); } std::hint::black_box(frame); } @@ -83,12 +123,17 @@ fn main() { let bytes_copy = start.elapsed(); println!( - "zero-copy: {:.3} ms/frame ({:.1} fps)", + "zero-copy: {:.3} ms/frame ({:.1} fps)", zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / zero_copy.as_secs_f64() ); println!( - "copy: {:.3} ms/frame ({:.1} fps)", + "preview copy: {:.3} ms/frame ({:.1} fps)", + preview_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / preview_copy.as_secs_f64() + ); + println!( + "row copy: {:.3} ms/frame ({:.1} fps)", copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / copy.as_secs_f64() ); diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 9873f1cc878..790c8c6084e 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr, sync::Arc}; +use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr}; use cap_camera::CapturedFrame; use cap_camera_windows::PixelFormat; @@ -242,6 +242,35 @@ struct MfPlane { stride: usize, } +struct MfPlanes { + planes: [MfPlane; 3], + len: usize, +} + +impl MfPlanes { + fn new() -> Self { + Self { + planes: [MfPlane { + ptr: ptr::null_mut(), + len: 0, + stride: 0, + }; 3], + len: 0, + } + } + + fn push(&mut self, plane: MfPlane) { + if self.len < self.planes.len() { + self.planes[self.len] = plane; + self.len += 1; + } + } + + fn as_slice(&self) -> &[MfPlane] { + &self.planes[..self.len] + } +} + enum MfLock { TwoD2(IMF2DBuffer2), TwoD(IMF2DBuffer), @@ -276,7 +305,7 @@ impl Drop for WrappedMfBuffer { unsafe extern "C" fn release_wrapped_mf_buffer(opaque: *mut c_void, _data: *mut u8) { if !opaque.is_null() { - drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + drop(unsafe { Box::from_raw(opaque as *mut WrappedMfBuffer) }); } } @@ -319,7 +348,7 @@ fn mf_planes( data: *mut u8, len: usize, stride: usize, -) -> Option> { +) -> Option { let row_bytes = mf_row_bytes(format, width)?; if stride < row_bytes || height == 0 { return None; @@ -327,27 +356,17 @@ fn mf_planes( let plane = |ptr: *mut u8, len: usize| MfPlane { ptr, len, stride }; let y_len = stride.checked_mul(height)?; + let mut planes = MfPlanes::new(); match format { - PixelFormat::NV12 | PixelFormat::NV21 => { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::P010 => { let uv_len = stride.checked_mul(height / 2)?; if len < y_len.checked_add(uv_len)? { return None; } - Some(vec![ - plane(data, y_len), - plane(unsafe { data.add(y_len) }, uv_len), - ]) - } - PixelFormat::P010 => { - let uv_len = stride.checked_mul(height / 2)?; - if len < y_len.checked_add(uv_len)? { - return None; - } - Some(vec![ - plane(data, y_len), - plane(unsafe { data.add(y_len) }, uv_len), - ]) + planes.push(plane(data, y_len)); + planes.push(plane(unsafe { data.add(y_len) }, uv_len)); + Some(planes) } PixelFormat::YUYV422 | PixelFormat::UYVY422 @@ -361,7 +380,8 @@ fn mf_planes( if len < y_len { return None; } - Some(vec![plane(data, y_len)]) + planes.push(plane(data, y_len)); + Some(planes) } PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => None, } @@ -372,7 +392,7 @@ fn lock_mf_buffer( format: PixelFormat, width: usize, height: usize, -) -> Option<(WrappedMfBuffer, Vec)> { +) -> Option<(WrappedMfBuffer, MfPlanes)> { unsafe { if let Ok(buffer_2d) = buffer.cast::() { let mut scanline = ptr::null_mut(); @@ -475,7 +495,7 @@ pub fn media_buffer_as_ffmpeg( let pixel = mf_pixel_format(format)?; let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; - wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) + wrap_mf_frame(&planes, pixel, width as u32, height as u32, owner) } // UYVY and NV21 use the format-converting copy paths, which lock with @@ -499,6 +519,7 @@ pub fn with_media_buffer_data( } let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); let total_len = planes.iter().map(|plane| plane.len).sum::(); let data = unsafe { std::slice::from_raw_parts(planes[0].ptr, total_len) }; let primary_stride = planes[0].stride; @@ -521,6 +542,7 @@ pub fn copy_media_buffer_to_ffmpeg( let pixel = mf_pixel_format(format)?; let row_bytes = mf_row_bytes(format, width)?; let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); let mut frame = FFVideo::new(pixel, width as u32, height as u32); for (index, plane) in planes.iter().enumerate() { @@ -541,12 +563,32 @@ pub fn copy_media_buffer_to_ffmpeg( } fn wrap_mf_frame( - planes: &[MfPlane], + planes: &MfPlanes, pixel: Pixel, width: u32, height: u32, - owner: Arc, + owner: WrappedMfBuffer, ) -> Option { + let planes = planes.as_slice(); + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let opaque = Box::into_raw(Box::new(owner)) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + planes[0].ptr, + total_len, + Some(release_wrapped_mf_buffer), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut WrappedMfBuffer)); + } + return None; + } + let mut frame = FFVideo::empty(); let ptr = unsafe { frame.as_mut_ptr() }; let format: ffi::AVPixelFormat = pixel.into(); @@ -554,37 +596,8 @@ fn wrap_mf_frame( (*ptr).format = format as i32; (*ptr).width = width as i32; (*ptr).height = height as i32; - } - - let mut buffers = Vec::with_capacity(planes.len()); - for plane in planes { - let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; - let buffer = unsafe { - ffi::av_buffer_create( - plane.ptr, - plane.len, - Some(release_wrapped_mf_buffer), - opaque, - ffi::AV_BUFFER_FLAG_READONLY, - ) - }; - - if buffer.is_null() { - unsafe { - drop(Box::from_raw(opaque as *mut Arc)); - for mut buffer in buffers { - ffi::av_buffer_unref(&mut buffer); - } - } - return None; - } - - buffers.push(buffer); - } - - for (index, (buffer, plane)) in buffers.into_iter().zip(planes).enumerate() { - unsafe { - (*ptr).buf[index] = buffer; + (*ptr).buf[0] = buffer; + for (index, plane) in planes.iter().enumerate() { (*ptr).data[index] = plane.ptr; (*ptr).linesize[index] = plane.stride as i32; } diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 2e496e47f64..e0fee8123d0 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -234,6 +234,10 @@ impl From for EncoderRuntimeError { unsafe impl Send for H264Encoder {} impl H264Encoder { + pub fn input_texture(&self) -> &ID3D11Texture2D { + self.video_processor.input_texture() + } + #[allow(clippy::too_many_arguments)] fn new_with_scaled_output_with_flags( d3d_device: &ID3D11Device, @@ -592,7 +596,12 @@ impl H264Encoder { } let process_result = (|| -> windows::core::Result<()> { - self.video_processor.process_texture(&texture)?; + if texture.as_raw() != self.video_processor.input_texture().as_raw() + { + self.video_processor.process_texture(&texture)?; + } else { + self.video_processor.process_input_texture()?; + } let input_buffer = MFCreateDXGISurfaceBuffer( &ID3D11Texture2D::IID, self.video_processor.output_texture(), diff --git a/crates/enc-mediafoundation/src/video/video_processor.rs b/crates/enc-mediafoundation/src/video/video_processor.rs index fe26d73f337..147532a3796 100644 --- a/crates/enc-mediafoundation/src/video/video_processor.rs +++ b/crates/enc-mediafoundation/src/video/video_processor.rs @@ -218,6 +218,10 @@ impl VideoProcessor { &self.video_output_texture } + pub fn input_texture(&self) -> &ID3D11Texture2D { + &self.video_input_texture + } + pub fn process_texture( &mut self, input_texture: &ID3D11Texture2D, @@ -226,18 +230,23 @@ impl VideoProcessor { // texture that matches the input size we were initialized with. unsafe { - // Copy the texture to the video input texture self.d3d_context .CopyResource(&self.video_input_texture, input_texture); + } - // Convert to NV12 - let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { - Enable: true.into(), - OutputIndex: 0, - InputFrameOrField: 0, - pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), - ..Default::default() - }; + self.process_input_texture() + } + + pub fn process_input_texture(&mut self) -> windows::core::Result<()> { + let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + OutputIndex: 0, + InputFrameOrField: 0, + pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), + ..Default::default() + }; + + unsafe { self.video_context.VideoProcessorBlt( &self.video_processor, &self.video_output, diff --git a/crates/recording/examples/camera-upload-bench.rs b/crates/recording/examples/camera-upload-bench.rs new file mode 100644 index 00000000000..e58c3762746 --- /dev/null +++ b/crates/recording/examples/camera-upload-bench.rs @@ -0,0 +1,130 @@ +#[cfg(windows)] +fn main() -> anyhow::Result<()> { + use cap_camera_ffmpeg::with_media_buffer_data; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_recording::{ + CameraBuffer, CameraBuffers, NativeCameraFrame, create_d3d_device, + upload_mf_buffer_to_texture, + }; + use cap_timestamp::Timestamp; + use std::time::Instant; + use windows::Win32::{ + Graphics::{ + Direct3D11::{ + D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, + }, + Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, + }, + Media::MediaFoundation::MFCreate2DMediaBuffer, + }; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 500; + + cap_mediafoundation_utils::thread_init(); + let device = create_d3d_device()?; + let buffer = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + }?; + + { + let mut lock = buffer.lock_for_write()?; + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 17) as u8; + } + } + + let frame = NativeCameraFrame { + buffer: CameraBuffer::new(buffer), + pixel_format: PixelFormat::NV12, + width: WIDTH as u32, + height: HEIGHT as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(Instant::now()), + }; + let mut buffers = CameraBuffers::new(); + + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + let elapsed = start.elapsed(); + + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: WIDTH as u32, + Height: HEIGHT as u32, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + + let start = Instant::now(); + with_media_buffer_data( + frame.buffer.get(), + PixelFormat::NV12, + WIDTH, + HEIGHT, + |data, pitch, _| { + for _ in 0..ITERATIONS { + let subresource = D3D11_SUBRESOURCE_DATA { + pSysMem: data.as_ptr() as *const _, + SysMemPitch: pitch as u32, + SysMemSlicePitch: 0, + }; + let mut texture = None; + unsafe { + device.CreateTexture2D( + &texture_desc, + Some(&subresource), + Some(&mut texture), + )?; + } + std::hint::black_box(texture); + } + + Ok::<_, windows::core::Error>(()) + }, + ) + .expect("locked data")?; + let uncached = start.elapsed(); + + println!( + "cached upload: {:.3} ms/frame ({:.1} fps)", + elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / elapsed.as_secs_f64() + ); + println!( + "uncached upload: {:.3} ms/frame ({:.1} fps)", + uncached.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / uncached.as_secs_f64() + ); + + Ok(()) +} + +#[cfg(not(windows))] +fn main() { + println!("camera-upload-bench requires Windows"); +} diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index ea6f82b3c2a..e90fc52cd2b 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -25,7 +25,7 @@ use tokio::{sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use crate::ffmpeg::FFmpegVideoFrame; -use crate::output_pipeline::{NativeCameraFrame, VideoFrame}; +use crate::output_pipeline::{CameraBuffer, NativeCameraFrame, VideoFrame}; const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); /// Outer deadline for camera readiness. Must cover both capture attempts on @@ -1270,8 +1270,7 @@ async fn setup_camera( use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; let buffer = if let Some(buffer) = frame.native().media_buffer() { - #[allow(clippy::arc_with_non_send_sync)] - Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + Some(CameraBuffer::new(buffer)) } else if let Ok(bytes) = frame.native().bytes() { let data_len = bytes.len(); unsafe { MFCreateMemoryBuffer(data_len as u32) } @@ -1285,8 +1284,7 @@ async fn setup_camera( }) .and_then(|buffer| { unsafe { buffer.SetCurrentLength(data_len as u32) }.ok()?; - #[allow(clippy::arc_with_non_send_sync)] - Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + Some(CameraBuffer::new(buffer)) }) } else { None diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index 341ed4c47f1..a20e3123b12 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -23,6 +23,8 @@ pub mod upload_preparation; pub mod upload_resume; pub mod upload_verification; +#[cfg(windows)] +pub use capture_pipeline::create_d3d_device; #[cfg(target_os = "linux")] pub use capture_pipeline::target_to_display_and_crop; pub use resolution_limits::{H264_MAX_DIMENSION, calculate_gpu_compatible_size}; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index c567a82e94a..9677349fab6 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -120,10 +120,13 @@ use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, Win32::Graphics::{ - Direct3D11::ID3D11Device, + Direct3D11::{ + D3D11_TEXTURE2D_DESC, ID3D11Device, ID3D11DeviceContext, ID3D11Resource, + ID3D11Texture2D, + }, Dxgi::Common::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_R8G8B8A8_UNORM, - DXGI_FORMAT_YUY2, + DXGI_FORMAT_UNKNOWN, DXGI_FORMAT_YUY2, }, }, }; @@ -506,11 +509,13 @@ impl Muxer for WindowsMuxer { #[cfg(test)] mod tests { - use super::{MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, reference_video_frame}; + use super::{ + CameraBuffer, MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, + reference_video_frame, + }; use cap_camera_windows::PixelFormat; use cap_mediafoundation_utils::IMFMediaBufferExt; use cap_timestamp::Timestamp; - use std::sync::{Arc, Mutex}; use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; #[test] @@ -535,9 +540,8 @@ mod tests { lock.as_ptr() }; - #[allow(clippy::arc_with_non_send_sync)] let frame = NativeCameraFrame { - buffer: Arc::new(Mutex::new(buffer)), + buffer: CameraBuffer::new(buffer), pixel_format: PixelFormat::NV12, width: width as u32, height: height as u32, @@ -644,10 +648,29 @@ fn duration_to_timespan(duration: Duration) -> TimeSpan { } } +type CameraMediaBuffer = windows::Win32::Media::MediaFoundation::IMFMediaBuffer; + +#[derive(Clone)] +pub struct CameraBuffer(CameraMediaBuffer); + +impl CameraBuffer { + pub fn new(buffer: CameraMediaBuffer) -> Self { + Self(buffer) + } + + pub fn get(&self) -> &CameraMediaBuffer { + &self.0 + } +} + +// Media Foundation buffers are free-threaded; pixel access is guarded by the +// per-frame 2D or contiguous lock held by the consumer. +unsafe impl Send for CameraBuffer {} +unsafe impl Sync for CameraBuffer {} + #[derive(Clone)] pub struct NativeCameraFrame { - pub buffer: - std::sync::Arc>, + pub buffer: CameraBuffer, pub pixel_format: cap_camera_windows::PixelFormat, pub width: u32, pub height: u32, @@ -921,7 +944,8 @@ impl Muxer for WindowsCameraMuxer { let mut last_frame: Option = None; let mut first_timestamp: Option = None; let mut frame_count = 0u64; - let mut camera_buffers = CameraBuffers::new(); + let mut camera_buffers = + CameraBuffers::with_texture(encoder.input_texture().clone()); let result = encoder.run( Arc::new(AtomicBool::default()), @@ -1141,6 +1165,11 @@ impl AudioMuxer for WindowsCameraMuxer { pub struct CameraBuffers { uyvy_buffer: Vec, flip_buffer: Vec, + texture: Option, + texture_width: u32, + texture_height: u32, + texture_format: DXGI_FORMAT, + context: Option, } impl CameraBuffers { @@ -1148,9 +1177,66 @@ impl CameraBuffers { Self { uyvy_buffer: Vec::new(), flip_buffer: Vec::new(), + texture: None, + texture_width: 0, + texture_height: 0, + texture_format: DXGI_FORMAT_UNKNOWN, + context: None, + } + } + + pub fn with_texture(texture: ID3D11Texture2D) -> Self { + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { texture.GetDesc(&mut desc) }; + + Self { + texture: Some(texture), + texture_width: desc.Width, + texture_height: desc.Height, + texture_format: desc.Format, + ..Self::new() } } + fn ensure_texture( + &mut self, + device: &ID3D11Device, + desc: &D3D11_TEXTURE2D_DESC, + ) -> windows::core::Result { + if self.texture.is_none() + || self.texture_width != desc.Width + || self.texture_height != desc.Height + || self.texture_format != desc.Format + { + let mut texture = None; + unsafe { device.CreateTexture2D(desc, None, Some(&mut texture))? }; + self.texture = texture; + self.texture_width = desc.Width; + self.texture_height = desc.Height; + self.texture_format = desc.Format; + } + + self.texture.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "CreateTexture2D succeeded but returned no texture", + ) + }) + } + + fn context(&mut self, device: &ID3D11Device) -> windows::core::Result { + if self.context.is_none() { + self.context = Some(unsafe { device.GetImmediateContext()? }); + } + + self.context.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "GetImmediateContext returned no context", + ) + }) + } + fn ensure_uyvy_capacity(&mut self, size: usize) -> &mut [u8] { if self.uyvy_buffer.len() < size { self.uyvy_buffer.resize(size, 0); @@ -1401,9 +1487,8 @@ pub(crate) fn camera_frame_to_ffmpeg( } if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) - && let Ok(buffer) = frame.buffer.lock() && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( - &buffer, + frame.buffer.get(), frame.pixel_format, frame.width as usize, frame.height as usize, @@ -1429,11 +1514,9 @@ pub(crate) fn camera_frame_to_ffmpeg( other => anyhow::bail!("Unsupported camera pixel format: {:?}", other), }; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1693,11 +1776,9 @@ fn decode_mjpeg_frame( ) -> anyhow::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1867,11 +1948,7 @@ pub fn upload_mf_buffer_to_texture( ) -> windows::core::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - let buffer_guard = frame - .buffer - .lock() - .map_err(|_| windows::core::Error::from(windows::core::HRESULT(-1)))?; - let buffer = &*buffer_guard; + let buffer = frame.buffer.get(); // The zero-copy and preview paths lock these formats with Lock2D; the // upload must use the same lock type to avoid MF_E_INVALIDREQUEST. @@ -1929,13 +2006,28 @@ fn upload_mf_buffer_data_to_texture( layout: CameraBufferLayout, buffers: &mut CameraBuffers, ) -> windows::core::Result { - use windows::Win32::Graphics::Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, - }; + use windows::Win32::Graphics::Direct3D11::{D3D11_BIND_SHADER_RESOURCE, D3D11_USAGE_DEFAULT}; use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; + use windows::core::Interface; let dxgi_format = frame.dxgi_format(); + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: frame.width, + Height: frame.height, + MipLevels: 1, + ArraySize: 1, + Format: dxgi_format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let texture: ID3D11Texture2D = buffers.ensure_texture(device, &texture_desc)?; + let context: ID3D11DeviceContext = buffers.context(device)?; let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; let needs_flip = frame.is_bottom_up; @@ -2021,36 +2113,11 @@ fn upload_mf_buffer_data_to_texture( } }; - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: frame.width, - Height: frame.height, - MipLevels: 1, - ArraySize: 1, - Format: dxgi_format, - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let subresource_data = D3D11_SUBRESOURCE_DATA { - pSysMem: data.as_ptr() as *const _, - SysMemPitch: row_pitch, - SysMemSlicePitch: 0, - }; + let resource = texture.cast::()?; unsafe { - let mut texture = None; - device.CreateTexture2D(&texture_desc, Some(&subresource_data), Some(&mut texture))?; - texture.ok_or_else(|| { - windows::core::Error::new( - windows::core::HRESULT(-1), - "CreateTexture2D succeeded but returned no texture", - ) - }) + context.UpdateSubresource(&resource, 0, None, data.as_ptr() as *const _, row_pitch, 0); } + + Ok(texture) } diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs index aad2d2af535..3b7b09dee7d 100644 --- a/crates/verify/tests/recording.rs +++ b/crates/verify/tests/recording.rs @@ -31,6 +31,8 @@ use cap_project::{ InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, XY, }; +#[cfg(windows)] +use cap_recording::CameraBuffer; use cap_recording::{ NativeCameraFrame, RecordingDefaults, StudioQuality, feeds::{ @@ -1183,10 +1185,8 @@ fn camera_frame(slot: u32, capture: u64) -> Option { let buffer = unsafe { MFCreateMemoryBuffer(nv12.len() as u32) }.ok()?; buffer.lock_for_write().ok()?.copy_from_slice(&nv12); unsafe { buffer.SetCurrentLength(nv12.len() as u32) }.ok()?; - #[allow(clippy::arc_with_non_send_sync)] - let buffer = Arc::new(std::sync::Mutex::new(buffer)); Some(NativeCameraFrame { - buffer, + buffer: CameraBuffer::new(buffer), pixel_format: cap_camera_windows::PixelFormat::NV12, width: w as u32, height: h as u32, From a5dd47eae3fec522ba84208ac2e359e858361397 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 07:40:28 +0100 Subject: [PATCH 4/7] fix(camera): restore non-Windows builds and benchmark direct upload Gate the Windows-only CameraBuffer import in feeds/camera.rs; the unconditional import broke macOS and Linux builds of cap-recording. Extend camera-upload-bench with the direct encoder-input path and a cached-upload-plus-CopyResource comparison. At 1080p NV12 the direct path is 1.053 ms/frame versus 1.359 ms/frame for the two-step path; cached and uncached uploads are 0.700 and 0.912 ms/frame. --- .../recording/examples/camera-upload-bench.rs | 53 ++++++++++++++++++- crates/recording/src/feeds/camera.rs | 4 +- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/recording/examples/camera-upload-bench.rs b/crates/recording/examples/camera-upload-bench.rs index e58c3762746..692d7858fb0 100644 --- a/crates/recording/examples/camera-upload-bench.rs +++ b/crates/recording/examples/camera-upload-bench.rs @@ -12,13 +12,14 @@ fn main() -> anyhow::Result<()> { use windows::Win32::{ Graphics::{ Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, + D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, ID3D11Resource, }, Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, }, Media::MediaFoundation::MFCreate2DMediaBuffer, }; + use windows::core::Interface; const WIDTH: usize = 1920; const HEIGHT: usize = 1080; @@ -110,6 +111,44 @@ fn main() -> anyhow::Result<()> { .expect("locked data")?; let uncached = start.elapsed(); + let encoder_desc = D3D11_TEXTURE2D_DESC { + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + ..texture_desc + }; + let mut encoder_texture = None; + unsafe { device.CreateTexture2D(&encoder_desc, None, Some(&mut encoder_texture))? }; + let encoder_texture = encoder_texture.expect("encoder input texture"); + let encoder_resource = encoder_texture.cast::()?; + let context = unsafe { device.GetImmediateContext()? }; + + let mut direct_buffers = CameraBuffers::with_texture(encoder_texture); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + let direct = start.elapsed(); + + let mut copy_buffers = CameraBuffers::new(); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + let cached_plus_copy = start.elapsed(); + println!( "cached upload: {:.3} ms/frame ({:.1} fps)", elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64, @@ -120,6 +159,16 @@ fn main() -> anyhow::Result<()> { uncached.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / uncached.as_secs_f64() ); + println!( + "direct upload: {:.3} ms/frame ({:.1} fps)", + direct.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / direct.as_secs_f64() + ); + println!( + "cached + copy: {:.3} ms/frame ({:.1} fps)", + cached_plus_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / cached_plus_copy.as_secs_f64() + ); Ok(()) } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index e90fc52cd2b..4af478ab372 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -25,7 +25,9 @@ use tokio::{sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use crate::ffmpeg::FFmpegVideoFrame; -use crate::output_pipeline::{CameraBuffer, NativeCameraFrame, VideoFrame}; +#[cfg(windows)] +use crate::output_pipeline::CameraBuffer; +use crate::output_pipeline::{NativeCameraFrame, VideoFrame}; const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); /// Outer deadline for camera readiness. Must cover both capture attempts on From 00864195c8bad018ec6894bf6ea0935edf416926 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 10:40:19 +0100 Subject: [PATCH 5/7] fix(camera): address Greptile review findings Infer the stride of contiguous camera buffers from the buffer length so padded DirectShow/MF memory buffers keep the correct chroma offset; add a padded NV12 regression test. Use cv::PixelFormat::to_cf_number() for the VideoToolbox pool attributes to match the other pool builders and keep the macOS converter building. --- crates/camera-ffmpeg/src/windows.rs | 80 +++++++++++++++++++++- crates/frame-converter/src/videotoolbox.rs | 2 +- 2 files changed, 78 insertions(+), 4 deletions(-) diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 790c8c6084e..4ef4aa6433a 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -341,6 +341,38 @@ fn mf_row_bytes(format: PixelFormat, width: usize) -> Option { }) } +fn contiguous_stride( + format: PixelFormat, + width: usize, + height: usize, + len: usize, +) -> Option { + let row_bytes = mf_row_bytes(format, width)?; + let total_rows = match format { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::P010 => { + height.checked_add(height / 2)? + } + PixelFormat::YUV420P | PixelFormat::YV12 => height.checked_add(height / 2)?, + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::ARGB + | PixelFormat::RGB32 + | PixelFormat::RGB24 + | PixelFormat::BGR24 + | PixelFormat::GRAY8 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 => height, + PixelFormat::MJPEG | PixelFormat::H264 => return None, + }; + + if total_rows == 0 || !len.is_multiple_of(total_rows) { + return None; + } + + let stride = len / total_rows; + (stride >= row_bytes).then_some(stride) +} + fn mf_planes( format: PixelFormat, width: usize, @@ -462,9 +494,8 @@ fn lock_mf_buffer( if buffer.Lock(&mut data, None, Some(&mut len)).is_ok() { if !data.is_null() && len > 0 - && let Some(row_bytes) = mf_row_bytes(format, width) - && let Some(planes) = - mf_planes(format, width, height, data, len as usize, row_bytes) + && let Some(stride) = contiguous_stride(format, width, height, len as usize) + && let Some(planes) = mf_planes(format, width, height, data, len as usize, stride) { return Some(( WrappedMfBuffer { @@ -1037,6 +1068,49 @@ mod conversion_tests { assert_eq!(frame.data(1)[stride + 2], 5); } + #[test] + fn padded_nv12_memory_buffer_uses_inferred_stride() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 96; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for row in 0..height { + for column in 0..width { + lock[row * stride + column] = (row + column) as u8; + } + } + let uv = stride * height; + for row in 0..height / 2 { + for column in 0..width { + lock[uv + row * stride + column] = (row * 3 + column) as u8; + } + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let base_ptr = { + let lock = buffer.lock().expect("read lock"); + lock.as_ptr() + }; + let frame = media_buffer_as_ffmpeg(&buffer, PixelFormat::NV12, width, height, false) + .expect("padded zero-copy frame"); + + assert_eq!(frame.stride(0), stride); + assert_eq!(frame.stride(1), stride); + assert_eq!(frame.data(0).as_ptr(), base_ptr); + assert_eq!(frame.data(1).as_ptr(), unsafe { + base_ptr.add(stride * height) + }); + assert_eq!(frame.data(0)[stride + 1], 2); + assert_eq!(frame.data(1)[stride + 2], 5); + } + #[test] fn as_ffmpeg_keeps_nv12_for_nv21_frames() { cap_mediafoundation_utils::thread_init(); diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index b16a3b65f42..fe03eb976de 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -71,7 +71,7 @@ fn create_pool( cv::pixel_buffer::keys::io_surf_props(), ], &[ - format.as_ref(), + format.to_cf_number().as_ref(), width.as_type_ref(), height.as_type_ref(), io_surface.as_type_ref(), From a269deea41c4afb830bbc7adb1309dc63a58e8e3 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 10:54:28 +0100 Subject: [PATCH 6/7] fix(camera): remove unused VideoToolbox pixel buffer alias --- crates/frame-converter/src/videotoolbox.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index fe03eb976de..ed60512ccf1 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -8,8 +8,6 @@ use std::{ sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; -type CVPixelBufferRef = *mut c_void; - #[link(name = "CoreVideo", kind = "framework")] unsafe extern "C" { fn CVPixelBufferGetBaseAddress(pixel_buffer: &cv::PixelBuf) -> *mut u8; From 09861afa2b2bf4a3f48b77f1564a32ec2ecc7704 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Thu, 5 Nov 2026 02:08:41 +0100 Subject: [PATCH 7/7] fix(camera): address macOS clippy let-and-return --- crates/camera-ffmpeg/src/macos.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index fa27b56a4a3..0ed6ff5f293 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -116,7 +116,7 @@ pub fn sample_buf_as_ffmpeg( cv::pixel_buffer::LockFlags::READ_ONLY, )?; - let res = match cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()) { + match cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()) { "2vuy" => { let mut ff_frame = ffmpeg::frame::Video::new( ffmpeg::format::Pixel::UYVY422, @@ -454,9 +454,7 @@ pub fn sample_buf_as_ffmpeg( return Err(AsFFmpegError::UnsupportedSubType(format.to_string())); } } - }; - - res + } }; CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64);