From c11ac012113c89f25fca93c1f4c3f1b02ab44583 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Sun, 25 Oct 2026 20:32:52 +0100 Subject: [PATCH 1/4] perf(camera): remove per-frame copies from capture path Wrap Media Foundation buffers and CVPixelBuffer planes as refcounted AVFrames instead of allocating and copying each frame. Keep native Media Foundation buffers through the feed, run MF and DirectShow sample callbacks on bounded worker queues, and reuse an IOSurface-backed CVPixelBufferPool for VideoToolbox output. Add conversion counters and a synthetic 1080p frame-copy benchmark. --- Cargo.lock | 3 +- crates/camera-directshow/src/lib.rs | 110 ++++- crates/camera-ffmpeg/Cargo.toml | 6 +- .../examples/frame-copy-bench.rs | 111 ++++++ crates/camera-ffmpeg/src/lib.rs | 55 +++ crates/camera-ffmpeg/src/linux.rs | 10 +- crates/camera-ffmpeg/src/macos.rs | 211 +++++++++- crates/camera-ffmpeg/src/windows.rs | 375 +++++++++++++++++- crates/camera-mediafoundation/Cargo.toml | 1 - crates/camera-mediafoundation/src/lib.rs | 100 ++++- crates/camera-windows/src/lib.rs | 68 +++- crates/camera/src/lib.rs | 9 + crates/camera/src/windows.rs | 8 +- crates/frame-converter/Cargo.toml | 1 + crates/frame-converter/src/videotoolbox.rs | 320 ++++++--------- crates/recording/src/feeds/camera.rs | 66 +-- crates/recording/src/output_pipeline/win.rs | 60 ++- 17 files changed, 1240 insertions(+), 274 deletions(-) create mode 100644 crates/camera-ffmpeg/examples/frame-copy-bench.rs diff --git a/Cargo.lock b/Cargo.lock index 5edf1df1581..6ff945d5749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1421,6 +1421,7 @@ dependencies = [ "cap-camera", "cap-camera-avfoundation", "cap-camera-windows", + "cap-mediafoundation-utils", "cidre", "ffmpeg-next", "inquire", @@ -1437,7 +1438,6 @@ version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", "inquire", - "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", @@ -1795,6 +1795,7 @@ name = "cap-frame-converter" version = "0.1.0" dependencies = [ "cap-media-info", + "cidre", "crossbeam-channel", "ffmpeg-next", "flume", diff --git a/crates/camera-directshow/src/lib.rs b/crates/camera-directshow/src/lib.rs index d2166053d68..3fb74fa08ad 100644 --- a/crates/camera-directshow/src/lib.rs +++ b/crates/camera-directshow/src/lib.rs @@ -8,7 +8,12 @@ use std::{ ops::Deref, os::windows::ffi::OsStringExt, ptr::{self, null, null_mut}, - sync::OnceLock, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel}, + }, + thread::{self, JoinHandle}, time::Duration, }; use tracing::*; @@ -460,7 +465,8 @@ impl VideoInputDevice { .SetFormat(&**format) .map_err(StartCapturingError::Other)?; - let sink_filter = SinkFilter::new(format.clone(), callback); + let (sink_filter, callback_worker) = SinkFilter::new(format.clone(), callback) + .map_err(StartCapturingError::CallbackWorker)?; let input_sink_pin = sink_filter .get_pin(0) @@ -515,6 +521,7 @@ impl VideoInputDevice { graph_builder, output_capture_pin: bound.output_pin, input_sink_pin, + callback_worker, }) } } @@ -525,6 +532,7 @@ pub struct CaptureHandle { graph_builder: IGraphBuilder, output_capture_pin: IPin, input_sink_pin: IPin, + callback_worker: CallbackWorker, } impl CaptureHandle { @@ -537,6 +545,8 @@ impl CaptureHandle { let _ = self.graph_builder.Disconnect(&self.input_sink_pin); } + drop(self.callback_worker); + Ok(()) } } @@ -553,6 +563,8 @@ pub enum StartCapturingError { ConfigureGraph(windows_core::Error), #[error("Run: {0}")] Run(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(std::io::Error), #[error("{0}")] Other(windows_core::Error), } @@ -628,13 +640,22 @@ pub struct SinkFilter { } impl SinkFilter { - pub fn new(desired_media_type: AMMediaType, callback: SinkCallback) -> ComObject { + pub(crate) fn new( + desired_media_type: AMMediaType, + callback: SinkCallback, + ) -> std::io::Result<(ComObject, CallbackWorker)> { + let (callback_tx, callback_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let dropped = Arc::new(AtomicU64::new(0)); + let callback_worker = + CallbackWorker::spawn(callback, callback_rx, Arc::new(AtomicBool::new(false)))?; + let this = SinkFilter { state: RefCell::new(State_Stopped), owning_graph: RefCell::new(None), input_pin: SinkInputPin { desired_media_type, - callback: RefCell::new(callback), + callback_tx, + dropped, current_media_type: Default::default(), connected_pin: Default::default(), owner: Default::default(), @@ -646,7 +667,7 @@ impl SinkFilter { // SAFETY: SinkFilter always implements IBaseFilter *this.input_pin.owner.borrow_mut() = Some(this.cast::().unwrap()); - this + Ok((this, callback_worker)) } fn no_of_pins(&self) -> u32 { @@ -814,7 +835,64 @@ pub struct CallbackData<'a> { pub perf_counter: i64, } -pub type SinkCallback = Box; +pub type SinkCallback = Box; + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct OwnedCallbackData { + sample: IMediaSample, + media_type: AMMediaType, + timestamp: Duration, + perf_counter: i64, +} + +unsafe impl Send for OwnedCallbackData {} + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: SinkCallback, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("dshow-camera-callback".to_string()) + .spawn(move || { + let _ = initialize_directshow(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(data) => callback(CallbackData { + sample: &data.sample, + media_type: &data.media_type, + timestamp: data.timestamp, + perf_counter: data.perf_counter, + }), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} #[implement(IPin, IMemInputPin)] struct SinkInputPin { @@ -823,7 +901,8 @@ struct SinkInputPin { current_media_type: RefCell, connected_pin: RefCell>, owner: RefCell>, - callback: RefCell, + callback_tx: SyncSender, + dropped: Arc, } // impl SinkInputPin { @@ -1029,7 +1108,7 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { } } - let media_type = self.current_media_type.borrow(); + let media_type = self.current_media_type.borrow().clone(); let length = unsafe { psample.GetActualDataLength() }; @@ -1046,12 +1125,19 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { unsafe { psample.GetTime(&mut start_time, &mut end_time) }?; - (self.callback.borrow_mut())(CallbackData { - sample: psample, - media_type: &media_type, + let data = OwnedCallbackData { + sample: psample.clone(), + media_type, timestamp: Duration::from_micros(start_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.callback_tx.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "DirectShow camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-ffmpeg/Cargo.toml b/crates/camera-ffmpeg/Cargo.toml index 6c98a31a488..6e7045baf25 100644 --- a/crates/camera-ffmpeg/Cargo.toml +++ b/crates/camera-ffmpeg/Cargo.toml @@ -16,9 +16,13 @@ cidre = { workspace = true } cap-camera-avfoundation = { path = "../camera-avfoundation" } [target.'cfg(windows)'.dependencies] -windows = { workspace = true } +windows = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Media_MediaFoundation", +] } windows-core = { workspace = true } cap-camera-windows = { path = "../camera-windows" } +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } [dev-dependencies] inquire = "0.7.5" diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs new file mode 100644 index 00000000000..9e9fe178b0d --- /dev/null +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -0,0 +1,111 @@ +#[cfg(windows)] +fn main() { + use cap_camera::CapturedFrame; + use cap_camera_ffmpeg::{CapturedFrameExt, conversion_stats}; + use cap_camera_windows::{Frame, PixelFormat}; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use ffmpeg::{format::Pixel, frame::Video}; + use std::time::{Duration, Instant}; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 1000; + + let len = WIDTH * HEIGHT * 3 / 2; + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + { + let mut lock = buffer.lock_for_write().expect("buffer lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("buffer length"); + + let frame = Frame::from_media_foundation( + buffer, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); + + for _ in 0..50 { + let frame = captured.as_ffmpeg().expect("warmup conversion"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = captured.as_ffmpeg().expect("conversion"); + std::hint::black_box(frame); + } + let zero_copy = start.elapsed(); + + let bytes = captured.native().bytes().expect("camera bytes"); + let start = Instant::now(); + for _ in 0..ITERATIONS { + let mut frame = Video::new(Pixel::NV12, WIDTH as u32, HEIGHT as u32); + let stride = frame.stride(0); + for row in 0..HEIGHT { + frame.data_mut(0)[row * stride..row * stride + WIDTH] + .copy_from_slice(&bytes[row * WIDTH..row * WIDTH + WIDTH]); + } + let uv_stride = frame.stride(1); + let uv_offset = WIDTH * HEIGHT; + for row in 0..HEIGHT / 2 { + frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH] + .copy_from_slice(&bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH]); + } + std::hint::black_box(frame); + } + let copy = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let buffer = captured.native().media_buffer().expect("native buffer"); + std::hint::black_box(buffer); + } + let buffer_clone = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let bytes = captured.native().bytes().expect("native bytes").to_vec(); + std::hint::black_box(bytes); + } + let bytes_copy = start.elapsed(); + + let stats = conversion_stats(); + println!( + "zero-copy: {:.3} ms/frame ({:.1} fps)", + zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / zero_copy.as_secs_f64() + ); + println!( + "copy: {:.3} ms/frame ({:.1} fps)", + copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / copy.as_secs_f64() + ); + println!( + "buffer clone: {:.3} ms/frame", + buffer_clone.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); + println!( + "bytes copy: {:.3} ms/frame", + bytes_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); + println!( + "stats: zero_copy={} copied={} avg_copy={:.3} us", + stats.zero_copy_frames, + stats.copied_frames, + stats.average_copy_micros() + ); +} + +#[cfg(not(windows))] +fn main() { + println!("frame-copy-bench requires Windows camera buffers"); +} diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index 3ec20823a6b..c13e638281a 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -13,6 +13,61 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Debug, Default, Clone, Copy)] +pub struct ConversionStatsSnapshot { + pub zero_copy_frames: u64, + pub copied_frames: u64, + pub copy_nanos: u64, +} + +impl ConversionStatsSnapshot { + pub fn average_copy_micros(&self) -> f64 { + if self.copied_frames == 0 { + 0.0 + } else { + self.copy_nanos as f64 / self.copied_frames as f64 / 1_000.0 + } + } +} + +#[derive(Debug, Default)] +pub struct ConversionStats { + zero_copy_frames: AtomicU64, + copied_frames: AtomicU64, + copy_nanos: AtomicU64, +} + +impl ConversionStats { + pub fn record_zero_copy(&self) { + self.zero_copy_frames.fetch_add(1, Ordering::Relaxed); + } + + pub fn record_copy(&self, elapsed_nanos: u64) { + self.copied_frames.fetch_add(1, Ordering::Relaxed); + self.copy_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); + } + + pub fn snapshot(&self) -> ConversionStatsSnapshot { + ConversionStatsSnapshot { + zero_copy_frames: self.zero_copy_frames.load(Ordering::Relaxed), + copied_frames: self.copied_frames.load(Ordering::Relaxed), + copy_nanos: self.copy_nanos.load(Ordering::Relaxed), + } + } +} + +pub static CONVERSION_STATS: ConversionStats = ConversionStats { + zero_copy_frames: AtomicU64::new(0), + copied_frames: AtomicU64::new(0), + copy_nanos: AtomicU64::new(0), +}; + +pub fn conversion_stats() -> ConversionStatsSnapshot { + CONVERSION_STATS.snapshot() +} + pub trait CapturedFrameExt { /// Creates an ffmpeg video frame from the native frame. /// Only size, format, and data are set. diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index 89e42e2a85b..62e303e995a 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -2,7 +2,7 @@ use cap_camera::{CapturedFrame, NativeFrameFormat}; use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; use std::cell::RefCell; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -77,8 +77,9 @@ impl CapturedFrameExt for CapturedFrame { let native = self.native(); let format = native.format; let bytes = native.bytes.as_slice(); + let started = std::time::Instant::now(); - match fourcc_str(format.fourcc).as_str() { + let frame = match fourcc_str(format.fourcc).as_str() { "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), @@ -88,7 +89,10 @@ impl CapturedFrameExt for CapturedFrame { "YV12" => copy_yuv420(bytes, format, true), "MJPG" | "JPEG" => decode_mjpeg(bytes), other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), - } + }?; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } } diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index a6c11b602bd..81f91919c42 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -1,13 +1,28 @@ use cap_camera::CapturedFrame; use cap_camera_avfoundation::ImageBufExt; use cidre::*; -use ffmpeg::{format::Pixel, software::scaling}; +use ffmpeg::{ffi, format::Pixel, software::scaling}; use std::{ cell::RefCell, - sync::atomic::{AtomicBool, Ordering}, + ffi::c_void, + ptr, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, }; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; + +type CVPixelBufferRef = *mut c_void; + +#[link(name = "CoreVideo", kind = "framework")] +unsafe extern "C" { + fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; + fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; + fn CVPixelBufferGetWidth(pixel_buffer: CVPixelBufferRef) -> usize; + fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; +} type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video); @@ -58,6 +73,181 @@ fn fourcc_to_pixel_format(fourcc: &str) -> Option { static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false); +#[link(name = "CoreVideo", kind = "framework")] +unsafe extern "C" { + fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; + fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; +} + +const CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 1; + +#[derive(Clone, Copy)] +struct SamplePlane { + ptr: *mut u8, + len: usize, + stride: usize, +} + +struct WrappedSampleBuf { + _image_buf: cidre::arc::R, + pixel_buffer: CVPixelBufferRef, + locked: bool, +} + +unsafe impl Send for WrappedSampleBuf {} +unsafe impl Sync for WrappedSampleBuf {} + +impl Drop for WrappedSampleBuf { + fn drop(&mut self) { + if self.locked { + unsafe { + CVPixelBufferUnlockBaseAddress(self.pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); + } + } + } +} + +unsafe extern "C" fn release_wrapped_sample_buf(opaque: *mut c_void, _data: *mut u8) { + if !opaque.is_null() { + drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + } +} + +fn sample_buf_planes( + image_buf: &cidre::cv::ImageBuf, + format: &str, +) -> Option<(Pixel, Vec, Option)> { + let (pixel, plane_count, range) = match format { + "2vuy" => (Pixel::UYVY422, 0, None), + "yuvs" => (Pixel::YUYV422, 0, None), + "BGRA" => (Pixel::BGRA, 0, None), + "ARGB" => (Pixel::ARGB, 0, None), + "24BG" => (Pixel::BGR24, 0, None), + "24RG" => (Pixel::RGB24, 0, None), + "L008" | "GRAY" => (Pixel::GRAY8, 0, None), + "L016" => (Pixel::GRAY16LE, 0, None), + "420v" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::MPEG)), + "420f" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::JPEG)), + "y420" => (Pixel::YUV420P, 3, None), + _ => return None, + }; + + if image_buf.plane_count() != plane_count { + return None; + } + + let mut planes = Vec::with_capacity(plane_count.max(1)); + if plane_count == 0 { + let pixel_buffer = image_buf as *const cidre::cv::ImageBuf as CVPixelBufferRef; + let ptr = unsafe { CVPixelBufferGetBaseAddress(pixel_buffer) }; + let stride = unsafe { CVPixelBufferGetBytesPerRow(pixel_buffer) }; + let height = unsafe { CVPixelBufferGetHeight(pixel_buffer) }; + if ptr.is_null() || stride == 0 || height == 0 { + return None; + } + planes.push(SamplePlane { + ptr, + len: stride.checked_mul(height)?, + stride, + }); + } else { + for plane in 0..plane_count { + let ptr = image_buf.plane_base_address(plane) as *mut u8; + let stride = image_buf.plane_bytes_per_row(plane); + let height = image_buf.plane_height(plane); + if ptr.is_null() || stride == 0 || height == 0 { + return None; + } + planes.push(SamplePlane { + ptr, + len: stride.checked_mul(height)?, + stride, + }); + } + } + + Some((pixel, planes, range)) +} + +fn wrap_sample_buf(sample_buf: &cidre::cm::SampleBuf) -> Option { + let image_buf = sample_buf.image_buf()?.retained(); + let format_desc = sample_buf.format_desc()?; + let format = cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()); + let pixel_buffer = (&*image_buf) as *const cidre::cv::ImageBuf as CVPixelBufferRef; + + if unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY) } != 0 { + return None; + } + + let Some((pixel, planes, range)) = sample_buf_planes(&image_buf, format) else { + unsafe { + CVPixelBufferUnlockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); + } + return None; + }; + + let owner = Arc::new(WrappedSampleBuf { + _image_buf: image_buf, + pixel_buffer, + locked: true, + }); + + let mut frame = ffmpeg::frame::Video::empty(); + let ptr = unsafe { frame.as_mut_ptr() }; + let format: ffi::AVPixelFormat = pixel.into(); + unsafe { + (*ptr).format = format as i32; + (*ptr).width = image_buf_width(&owner) as i32; + (*ptr).height = image_buf_height(&owner) as i32; + } + + let mut buffers = Vec::with_capacity(planes.len()); + for plane in &planes { + let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + plane.ptr, + plane.len, + Some(release_wrapped_sample_buf), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut Arc)); + for mut buffer in buffers { + ffi::av_buffer_unref(&mut buffer); + } + } + return None; + } + buffers.push(buffer); + } + + for (index, (buffer, plane)) in buffers.into_iter().zip(&planes).enumerate() { + unsafe { + (*ptr).buf[index] = buffer; + (*ptr).data[index] = plane.ptr; + (*ptr).linesize[index] = plane.stride as i32; + } + } + + if let Some(range) = range { + frame.set_color_range(range); + } + + Some(frame) +} + +fn image_buf_width(owner: &WrappedSampleBuf) -> usize { + unsafe { CVPixelBufferGetWidth(owner.pixel_buffer) } +} + +fn image_buf_height(owner: &WrappedSampleBuf) -> usize { + unsafe { CVPixelBufferGetHeight(owner.pixel_buffer) } +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) @@ -69,7 +259,13 @@ impl CapturedFrameExt for CapturedFrame { pub fn sample_buf_as_ffmpeg( sample_buf: &cm::SampleBuf, ) -> Result { - { + if let Some(frame) = wrap_sample_buf(sample_buf) { + CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + + let started = std::time::Instant::now(); + let frame = { let mut image_buf = sample_buf .image_buf() .map(|b| b.retained()) @@ -447,6 +643,9 @@ pub fn sample_buf_as_ffmpeg( } }; - Ok(res) - } + res + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 3b1ee47581c..34efbfcf89c 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -1,10 +1,16 @@ -use std::{cell::RefCell, collections::VecDeque}; +use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr, sync::Arc}; use cap_camera::CapturedFrame; use cap_camera_windows::PixelFormat; -use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; +use ffmpeg::{Packet, ffi, format::Pixel, frame::Video as FFVideo}; +use windows::{ + Win32::Media::MediaFoundation::{ + IMF2DBuffer, IMF2DBuffer2, IMFMediaBuffer, MF2DBuffer_LockFlags_Read, + }, + core::Interface, +}; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -229,6 +235,306 @@ fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize { if is_bottom_up { height - y - 1 } else { y } } +#[derive(Clone, Copy)] +struct MfPlane { + ptr: *mut u8, + len: usize, + stride: usize, +} + +enum MfLock { + TwoD2(IMF2DBuffer2), + TwoD(IMF2DBuffer), + Contiguous, +} + +struct WrappedMfBuffer { + _buffer: IMFMediaBuffer, + lock: MfLock, +} + +unsafe impl Send for WrappedMfBuffer {} +unsafe impl Sync for WrappedMfBuffer {} + +impl Drop for WrappedMfBuffer { + fn drop(&mut self) { + unsafe { + match &self.lock { + MfLock::TwoD2(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::TwoD(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::Contiguous => { + let _ = self._buffer.Unlock(); + } + } + } + } +} + +unsafe extern "C" fn release_wrapped_mf_buffer(opaque: *mut c_void, _data: *mut u8) { + if !opaque.is_null() { + drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + } +} + +fn mf_pixel_format(format: PixelFormat) -> Option { + Some(match format { + PixelFormat::NV12 => Pixel::NV12, + PixelFormat::NV21 => Pixel::NV21, + PixelFormat::YUYV422 => Pixel::YUYV422, + PixelFormat::UYVY422 => Pixel::UYVY422, + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => { + return None; + } + PixelFormat::ARGB | PixelFormat::RGB32 => Pixel::BGRA, + PixelFormat::RGB24 | PixelFormat::BGR24 => Pixel::BGR24, + PixelFormat::GRAY8 => Pixel::GRAY8, + PixelFormat::GRAY16 => Pixel::GRAY16LE, + PixelFormat::RGB565 => Pixel::RGB565LE, + PixelFormat::P010 => Pixel::P010LE, + }) +} + +fn mf_row_bytes(format: PixelFormat, width: usize) -> Option { + Some(match format { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::YUV420P | PixelFormat::GRAY8 => width, + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 + | PixelFormat::P010 => width.checked_mul(2)?, + PixelFormat::RGB24 | PixelFormat::BGR24 => width.checked_mul(3)?, + PixelFormat::ARGB | PixelFormat::RGB32 => width.checked_mul(4)?, + PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => return None, + }) +} + +fn mf_planes( + format: PixelFormat, + width: usize, + height: usize, + data: *mut u8, + len: usize, + stride: usize, +) -> Option> { + let row_bytes = mf_row_bytes(format, width)?; + if stride < row_bytes || height == 0 { + return None; + } + + let plane = |ptr: *mut u8, len: usize| MfPlane { ptr, len, stride }; + let y_len = stride.checked_mul(height)?; + + match format { + PixelFormat::NV12 | PixelFormat::NV21 => { + let uv_len = stride.checked_mul(height / 2)?; + if len < y_len.checked_add(uv_len)? { + return None; + } + Some(vec![ + plane(data, y_len), + plane(unsafe { data.add(y_len) }, uv_len), + ]) + } + PixelFormat::P010 => { + let uv_len = stride.checked_mul(height / 2)?; + if len < y_len.checked_add(uv_len)? { + return None; + } + Some(vec![ + plane(data, y_len), + plane(unsafe { data.add(y_len) }, uv_len), + ]) + } + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::ARGB + | PixelFormat::RGB32 + | PixelFormat::RGB24 + | PixelFormat::BGR24 + | PixelFormat::GRAY8 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 => { + if len < y_len { + return None; + } + Some(vec![plane(data, y_len)]) + } + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => None, + } +} + +fn lock_mf_buffer( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option<(WrappedMfBuffer, Vec, Pixel)> { + let pixel = mf_pixel_format(format)?; + + unsafe { + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + let mut start = ptr::null_mut(); + let mut len = 0u32; + if buffer_2d + .Lock2DSize( + MF2DBuffer_LockFlags_Read, + &mut scanline, + &mut pitch, + &mut start, + &mut len, + ) + .is_ok() + { + if !start.is_null() + && pitch > 0 + && len > 0 + && let Some(planes) = + mf_planes(format, width, height, start, len as usize, pitch as usize) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD2(buffer_2d), + }, + planes, + pixel, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + if buffer_2d.Lock2D(&mut scanline, &mut pitch).is_ok() { + if !scanline.is_null() + && pitch > 0 + && let Ok(len) = buffer_2d.GetContiguousLength() + && let Some(planes) = mf_planes( + format, + width, + height, + scanline, + len as usize, + pitch as usize, + ) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD(buffer_2d), + }, + planes, + pixel, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + let mut data = ptr::null_mut(); + let mut len = 0u32; + if buffer.Lock(&mut data, None, Some(&mut len)).is_ok() { + if !data.is_null() + && len > 0 + && let Some(row_bytes) = mf_row_bytes(format, width) + && let Some(planes) = + mf_planes(format, width, height, data, len as usize, row_bytes) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::Contiguous, + }, + planes, + pixel, + )); + } + + let _ = buffer.Unlock(); + } + } + + None +} + +pub fn media_buffer_as_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, +) -> Option { + if is_bottom_up || width == 0 || height == 0 { + return None; + } + + let (owner, planes, pixel) = lock_mf_buffer(buffer, format, width, height)?; + wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) +} + +fn wrap_mf_frame( + planes: &[MfPlane], + pixel: Pixel, + width: u32, + height: u32, + owner: Arc, +) -> Option { + let mut frame = FFVideo::empty(); + let ptr = unsafe { frame.as_mut_ptr() }; + let format: ffi::AVPixelFormat = pixel.into(); + unsafe { + (*ptr).format = format as i32; + (*ptr).width = width as i32; + (*ptr).height = height as i32; + } + + let mut buffers = Vec::with_capacity(planes.len()); + for plane in planes { + let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + plane.ptr, + plane.len, + Some(release_wrapped_mf_buffer), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut Arc)); + for mut buffer in buffers { + ffi::av_buffer_unref(&mut buffer); + } + } + return None; + } + + buffers.push(buffer); + } + + for (index, (buffer, plane)) in buffers.into_iter().zip(planes).enumerate() { + unsafe { + (*ptr).buf[index] = buffer; + (*ptr).data[index] = plane.ptr; + (*ptr).linesize[index] = plane.stride as i32; + } + } + + Some(frame) +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { let native = self.native(); @@ -240,13 +546,22 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } + if let Some(buffer) = native.media_buffer() + && let Some(frame) = + media_buffer_as_ffmpeg(&buffer, native.pixel_format, width, height, is_bottom_up) + { + CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + let bytes = native.bytes().map_err(AsFFmpegError::FailedToGetBytes)?; if bytes.is_empty() { return Err(AsFFmpegError::Empty); } - Ok(match native.pixel_format { + let started = std::time::Instant::now(); + let frame = match native.pixel_format { PixelFormat::YUV420P => { let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32); @@ -502,6 +817,56 @@ impl CapturedFrameExt for CapturedFrame { ff_frame } PixelFormat::H264 => decode_h264(&bytes)?, - }) + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) + } +} + +#[cfg(test)] +mod zero_copy_tests { + use super::*; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + #[test] + fn nv12_media_buffer_is_wrapped_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for row in 0..height { + for column in 0..width { + lock[row * stride + column] = (row + column) as u8; + } + } + let uv = stride * height; + for row in 0..height / 2 { + for column in 0..width { + lock[uv + row * stride + column] = (row * 3 + column) as u8; + } + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = media_buffer_as_ffmpeg(&buffer, PixelFormat::NV12, width, height, false) + .expect("zero-copy frame"); + + assert_eq!(frame.format(), Pixel::NV12); + assert_eq!(frame.width(), width as u32); + assert_eq!(frame.height(), height as u32); + assert_eq!(frame.stride(0), stride); + assert_eq!(frame.stride(1), stride); + assert_eq!(frame.data(0)[0], 0); + assert_eq!(frame.data(0)[stride + 1], 2); + assert_eq!(frame.data(1)[0], 0); + assert_eq!(frame.data(1)[stride + 2], 5); } } diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 36ee6fdd9d0..0bee6fe27df 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -7,7 +7,6 @@ license = "MIT" [dependencies] tracing.workspace = true thiserror.workspace = true -parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index 246b1c311e9..e09fa426e21 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -2,7 +2,6 @@ #![allow(non_snake_case)] use cap_mediafoundation_utils::*; -use parking_lot::Mutex; use std::{ ffi::OsString, fmt::Display, @@ -11,9 +10,13 @@ use std::{ os::windows::ffi::OsStringExt, slice::from_raw_parts, sync::{ - OnceLock, - mpsc::{Receiver, Sender, channel}, + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{ + Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError, channel, sync_channel, + }, }, + thread::{self, JoinHandle}, time::Duration, }; use windows::Win32::{ @@ -144,6 +147,8 @@ pub enum StartCapturingError { ConfigureSink(windows_core::Error), #[error("StartPreview: {0}")] StartPreview(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(#[from] std::io::Error), } impl Device { @@ -233,7 +238,7 @@ impl Device { pub fn start_capturing( &self, requested_format: &IMFMediaType, - callback: Box, + callback: Box, ) -> Result { let media_source = self .media_source() @@ -252,9 +257,14 @@ impl Device { .map_err(StartCapturingError::CreateEngine)?; let (event_tx, event_rx) = channel(); + let (sample_tx, sample_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let dropped = Arc::new(AtomicU64::new(0)); + let callback_worker = + CallbackWorker::spawn(callback, sample_rx, Arc::new(AtomicBool::new(false)))?; let video_callback = VideoCallback { event_tx, - sample_callback: Mutex::new(callback), + samples: sample_tx, + dropped, } .into_object(); @@ -375,7 +385,11 @@ impl Device { wait_for_event(&event_rx, CaptureEngineEventVariant::PreviewStarted) .map_err(|v| StartCapturingError::StartPreview(v.into()))?; - Ok(CaptureHandle { engine, event_rx }) + Ok(CaptureHandle { + engine: Some(engine), + event_rx, + callback_worker, + }) } } } @@ -408,8 +422,9 @@ fn retry_on_invalid_request( } pub struct CaptureHandle { + engine: Option, event_rx: Receiver, - engine: IMFCaptureEngine, + callback_worker: CallbackWorker, } impl CaptureHandle { @@ -417,8 +432,14 @@ impl CaptureHandle { &self.event_rx } - pub fn stop_capturing(self) -> windows_core::Result<()> { - unsafe { self.engine.StopPreview() } + pub fn stop_capturing(mut self) -> windows_core::Result<()> { + let result = if let Some(engine) = self.engine.take() { + unsafe { engine.StopPreview() } + } else { + Ok(()) + }; + drop(self.callback_worker); + result } } @@ -570,10 +591,56 @@ pub struct CallbackData { pub perf_counter: i64, } +unsafe impl Send for CallbackData {} + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: Box, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("mf-camera-callback".to_string()) + .spawn(move || { + cap_mediafoundation_utils::thread_init(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(sample) => callback(sample), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + #[implement(IMFCaptureEngineOnSampleCallback, IMFCaptureEngineOnEventCallback)] struct VideoCallback { event_tx: Sender, - sample_callback: Mutex>, + samples: SyncSender, + dropped: Arc, } impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { @@ -585,15 +652,20 @@ impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { return Ok(()); }; - let mut callback = self.sample_callback.lock(); - let sample_time = unsafe { sample.GetSampleTime() }?; - (callback)(CallbackData { + let data = CallbackData { sample: sample.clone(), timestamp: Duration::from_micros(sample_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.samples.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "Media Foundation camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index b1004ae2d6c..526a279ffd6 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -236,8 +236,10 @@ impl VideoDeviceInfo { .max_by_key(|(w, h)| (*w as u64) * (*h as u64)) } - pub fn find_best_format(&self, preference: &FormatPreference) -> Option { - let formats = self.formats(); + fn best_format_in( + formats: &[VideoFormat], + preference: &FormatPreference, + ) -> Option { if formats.is_empty() { return None; } @@ -267,11 +269,19 @@ impl VideoDeviceInfo { format_priority + resolution_score + fps_score }; - formats.into_iter().max_by_key(score_format) + formats + .iter() + .max_by_key(|format| score_format(format)) + .cloned() + } + + pub fn find_best_format(&self, preference: &FormatPreference) -> Option { + Self::best_format_in(&self.formats(), preference) } pub fn find_format_with_fallback(&self, preference: &FormatPreference) -> Option { - if let Some(format) = self.find_best_format(preference) { + let formats = self.formats(); + if let Some(format) = Self::best_format_in(&formats, preference) { return Some(format); } @@ -284,7 +294,6 @@ impl VideoDeviceInfo { PixelFormat::YUV420P, ]; - let formats = self.formats(); for fallback_pixel_format in fallback_formats { if let Some(format) = formats .iter() @@ -319,7 +328,7 @@ impl VideoDeviceInfo { pub fn start_capturing( self, format: &VideoFormatInner, - mut callback: impl FnMut(Frame) + 'static, + mut callback: impl FnMut(Frame) + Send + 'static, ) -> Result { let res = match (self.inner, &format) { ( @@ -327,6 +336,10 @@ impl VideoDeviceInfo { VideoFormatInner::MediaFoundation(mf_format), ) => { let format = VideoFormat::new_mf(mf_format.clone())?; + let width = format.width() as usize; + let height = format.height() as usize; + let is_bottom_up = format.is_bottom_up; + let pixel_format = format.pixel_format; let handle = device.start_capturing( mf_format, @@ -340,10 +353,10 @@ impl VideoDeviceInfo { callback(Frame { inner: FrameInner::MediaFoundation(buffer), - width: format.width() as usize, - height: format.height() as usize, - is_bottom_up: format.is_bottom_up, - pixel_format: format.pixel_format, + width, + height, + is_bottom_up, + pixel_format, timestamp: data.timestamp, perf_counter: data.perf_counter, }) @@ -476,6 +489,41 @@ impl Frame { FrameInner::MediaFoundation(buffer) => Ok(FrameBytes::MediaFoundation(buffer.lock()?)), } } + + pub fn media_buffer(&self) -> Option { + match &self.inner { + FrameInner::MediaFoundation(buffer) => Some(buffer.clone()), + FrameInner::DirectShow(_) => None, + } + } + + pub fn directshow_sample(&self) -> Option { + match &self.inner { + FrameInner::DirectShow(sample) => Some(sample.clone()), + FrameInner::MediaFoundation(_) => None, + } + } + + #[doc(hidden)] + pub fn from_media_foundation( + buffer: IMFMediaBuffer, + pixel_format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, + timestamp: Duration, + perf_counter: i64, + ) -> Self { + Self { + pixel_format, + width, + height, + is_bottom_up, + timestamp, + perf_counter, + inner: FrameInner::MediaFoundation(buffer), + } + } } pub enum FrameBytes<'a> { diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index 98b2875060b..c476d78f5f9 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -230,6 +230,15 @@ impl CapturedFrame { pub fn native(&self) -> &NativeCapturedFrame { &self.native } + + #[cfg(windows)] + #[doc(hidden)] + pub fn from_windows_frame(frame: cap_camera_windows::Frame, timestamp: Duration) -> Self { + Self { + native: NativeCapturedFrame::from_frame(frame), + timestamp, + } + } } impl CameraInfo { diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index a99d3d7b81f..07d72fa184d 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -64,12 +64,18 @@ impl ModelID { #[derive(Debug)] pub struct NativeCapturedFrame(cap_camera_windows::Frame); +impl NativeCapturedFrame { + pub(crate) fn from_frame(frame: cap_camera_windows::Frame) -> Self { + Self(frame) + } +} + pub type NativeCaptureHandle = WindowsCaptureHandle; pub(super) fn start_capturing_impl( camera: &CameraInfo, format: Format, - mut callback: impl FnMut(CapturedFrame) + 'static, + mut callback: impl FnMut(CapturedFrame) + Send + 'static, ) -> Result { let device = find_device(camera)?.ok_or(StartCapturingError::DeviceNotFound)?; diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index 3d363ef7098..be104b4593f 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -19,6 +19,7 @@ parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "macos")'.dependencies] +cidre = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows = { workspace = true, features = [ diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index 8d4d8e28d87..e0a104cd95e 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -1,4 +1,5 @@ use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; +use cidre::{arc, cf, cv, os, vt}; use ffmpeg::{format::Pixel, frame}; use parking_lot::Mutex; use std::{ @@ -7,100 +8,43 @@ use std::{ sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; -type CFAllocatorRef = *const c_void; -type CFDictionaryRef = *const c_void; type CVPixelBufferRef = *mut c_void; -type VTPixelTransferSessionRef = *mut c_void; -type OSStatus = i32; - -const K_CV_RETURN_SUCCESS: i32 = 0; - -const K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8: u32 = 0x79757679; -const K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE: u32 = 0x34323076; -const K_CV_PIXEL_FORMAT_TYPE_2VUY: u32 = 0x32767579; -const K_CV_PIXEL_FORMAT_TYPE_32_BGRA: u32 = 0x42475241; -const K_CV_PIXEL_FORMAT_TYPE_32_ARGB: u32 = 0x00000020; - -#[link(name = "CoreFoundation", kind = "framework")] -unsafe extern "C" { - fn CFRelease(cf: *const c_void); -} #[link(name = "CoreVideo", kind = "framework")] unsafe extern "C" { - fn CVPixelBufferCreate( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferCreateWithBytes( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - base_address: *mut c_void, - bytes_per_row: usize, - release_callback: *const c_void, - release_ref_con: *const c_void, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferRelease(pixel_buffer: CVPixelBufferRef); - - fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - - fn CVPixelBufferGetBaseAddressOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> *mut u8; - fn CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetHeightOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetPlaneCount(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; - fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; -} - -#[link(name = "VideoToolbox", kind = "framework")] -unsafe extern "C" { - fn VTPixelTransferSessionCreate( - allocator: CFAllocatorRef, - pixel_transfer_session_out: *mut VTPixelTransferSessionRef, - ) -> OSStatus; - - fn VTPixelTransferSessionInvalidate(session: VTPixelTransferSessionRef); - - fn VTPixelTransferSessionTransferImage( - session: VTPixelTransferSessionRef, - source_buffer: CVPixelBufferRef, - destination_buffer: CVPixelBufferRef, - ) -> OSStatus; + fn CVPixelBufferGetBaseAddress(pixel_buffer: &cv::PixelBuf) -> *mut u8; + fn CVPixelBufferGetBytesPerRow(pixel_buffer: &cv::PixelBuf) -> usize; + fn CVPixelBufferGetHeight(pixel_buffer: &cv::PixelBuf) -> usize; } -fn pixel_to_cv_format(pixel: Pixel) -> Option { - match pixel { - Pixel::YUYV422 => Some(K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8), - Pixel::UYVY422 => Some(K_CV_PIXEL_FORMAT_TYPE_2VUY), - Pixel::NV12 => Some(K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE), - Pixel::BGRA => Some(K_CV_PIXEL_FORMAT_TYPE_32_BGRA), - Pixel::ARGB => Some(K_CV_PIXEL_FORMAT_TYPE_32_ARGB), - _ => None, - } +fn pixel_to_cv_format(pixel: Pixel) -> Option { + Some(match pixel { + Pixel::YUYV422 => cv::PixelFormat(os::Type::from_be_bytes(*b"yuvs")), + Pixel::UYVY422 => cv::PixelFormat::_2VUY, + Pixel::NV12 => cv::PixelFormat::_420V, + Pixel::BGRA => cv::PixelFormat::_32_BGRA, + Pixel::ARGB => cv::PixelFormat::_32_ARGB, + _ => return None, + }) } -struct SessionHandle(VTPixelTransferSessionRef); +struct SessionHandle(arc::R); unsafe impl Send for SessionHandle {} +unsafe impl Sync for SessionHandle {} + +struct PoolHandle(arc::R); + +unsafe impl Send for PoolHandle {} +unsafe impl Sync for PoolHandle {} pub struct VideoToolboxConverter { session: Mutex, + pool: Option, input_format: Pixel, - input_cv_format: u32, + input_cv_format: cv::PixelFormat, output_format: Pixel, - output_cv_format: u32, + output_cv_format: cv::PixelFormat, input_width: u32, input_height: u32, output_width: u32, @@ -109,6 +53,39 @@ pub struct VideoToolboxConverter { verified_hardware: AtomicBool, } +extern "C" fn noop_release(_: *mut c_void, _: *const *const c_void) {} + +fn create_pool( + width: u32, + height: u32, + format: cv::PixelFormat, +) -> Result { + let width = cf::Number::from_usize(width as usize); + let height = cf::Number::from_usize(height as usize); + let io_surface = cf::Dictionary::new(); + let attrs = cf::DictionaryOf::::with_keys_values( + &[ + cv::pixel_buffer::keys::pixel_format(), + cv::pixel_buffer::keys::width(), + cv::pixel_buffer::keys::height(), + cv::pixel_buffer::keys::io_surf_props(), + ], + &[ + format.as_ref(), + width.as_type_ref(), + height.as_type_ref(), + io_surface.as_type_ref(), + ], + ) + .ok_or_else(|| { + ConvertError::HardwareUnavailable("VideoToolbox pixel buffer pool attributes".to_string()) + })?; + + cv::PixelBufPool::new(None, Some(&**attrs)) + .map(PoolHandle) + .map_err(|error| ConvertError::HardwareUnavailable(format!("CVPixelBufferPool: {error}"))) +} + impl VideoToolboxConverter { pub fn new(config: ConversionConfig) -> Result { let input_cv_format = pixel_to_cv_format(config.input_format).ok_or( @@ -119,20 +96,18 @@ impl VideoToolboxConverter { ConvertError::UnsupportedFormat(config.input_format, config.output_format), )?; - let mut session: VTPixelTransferSessionRef = ptr::null_mut(); - let status = unsafe { VTPixelTransferSessionCreate(ptr::null(), &mut session) }; - - if status != 0 { - return Err(ConvertError::HardwareUnavailable(format!( - "VTPixelTransferSessionCreate failed with status: {status}" - ))); - } + let mut session = vt::PixelTransferSession::new().map_err(|error| { + ConvertError::HardwareUnavailable(format!("VTPixelTransferSessionCreate: {error}")) + })?; + let _ = session.set_realtime(true); - if session.is_null() { - return Err(ConvertError::HardwareUnavailable( - "VTPixelTransferSessionCreate returned null session".to_string(), - )); - } + let pool = match create_pool(config.output_width, config.output_height, output_cv_format) { + Ok(pool) => Some(pool), + Err(error) => { + tracing::debug!("VideoToolbox pixel buffer pool unavailable: {error}"); + None + } + }; tracing::debug!( "VideoToolbox converter initialized: {:?} {}x{} -> {:?} {}x{}", @@ -146,6 +121,7 @@ impl VideoToolboxConverter { Ok(Self { session: Mutex::new(SessionHandle(session)), + pool, input_format: config.input_format, input_cv_format, output_format: config.output_format, @@ -162,119 +138,100 @@ impl VideoToolboxConverter { fn create_input_pixel_buffer( &self, input: &frame::Video, - ) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - + ) -> Result, ConvertError> { let base_address = input.data(0).as_ptr() as *mut c_void; let bytes_per_row = input.stride(0); - let status = unsafe { - CVPixelBufferCreateWithBytes( - ptr::null(), - self.input_width as usize, - self.input_height as usize, - self.input_cv_format, - base_address, - bytes_per_row, - ptr::null(), - ptr::null(), - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreateWithBytes failed: {status}" - ))); - } - - Ok(pixel_buffer) + cv::PixelBuf::with_bytes( + self.input_width as usize, + self.input_height as usize, + base_address, + bytes_per_row, + noop_release, + ptr::null_mut(), + self.input_cv_format, + None, + ) + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferCreateWithBytes: {error}")) + }) } - fn create_output_pixel_buffer(&self) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - - let status = unsafe { - CVPixelBufferCreate( - ptr::null(), - self.output_width as usize, - self.output_height as usize, - self.output_cv_format, - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreate failed: {status}" - ))); + fn create_output_pixel_buffer(&self) -> Result, ConvertError> { + if let Some(pool) = &self.pool + && let Ok(buffer) = pool.0.pixel_buf() + { + return Ok(buffer); } - Ok(pixel_buffer) + cv::PixelBuf::new( + self.output_width as usize, + self.output_height as usize, + self.output_cv_format, + None, + ) + .map_err(|error| ConvertError::ConversionFailed(format!("CVPixelBufferCreate: {error}"))) } fn copy_output_to_frame( &self, - pixel_buffer: CVPixelBufferRef, + pixel_buffer: &mut cv::PixelBuf, output: &mut frame::Video, ) -> Result<(), ConvertError> { - unsafe { - let lock_status = CVPixelBufferLockBaseAddress(pixel_buffer, 0); - if lock_status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferLockBaseAddress failed: {lock_status}" - ))); - } - } + unsafe { pixel_buffer.lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY) } + .result() + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferLockBaseAddress: {error}")) + })?; unsafe { - let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + let plane_count = pixel_buffer.plane_count(); if plane_count == 0 { let src_ptr = CVPixelBufferGetBaseAddress(pixel_buffer); let src_stride = CVPixelBufferGetBytesPerRow(pixel_buffer); let height = CVPixelBufferGetHeight(pixel_buffer); let dst_stride = output.stride(0); - - let dst_data = output.data_mut(0); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(0).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } else { for plane in 0..plane_count { - let src_ptr = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane); - let src_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); - let height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + let src_ptr = pixel_buffer.plane_base_address(plane); + let src_stride = pixel_buffer.plane_bytes_per_row(plane); + let height = pixel_buffer.plane_height(plane); let dst_stride = output.stride(plane); - - let dst_data = output.data_mut(plane); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(plane).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } } + } - CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + unsafe { + let _ = pixel_buffer.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY); } Ok(()) @@ -283,12 +240,9 @@ impl VideoToolboxConverter { impl Drop for VideoToolboxConverter { fn drop(&mut self) { - let session = self.session.get_mut().0; - if !session.is_null() { - unsafe { - VTPixelTransferSessionInvalidate(session); - CFRelease(session as *const c_void); - } + self.session.get_mut().0.invalidate(); + if let Some(pool) = &self.pool { + pool.0.flush(cv::PixelBufPoolFlushFlags::EXCESS_BUFFERS); } } } @@ -317,25 +271,17 @@ impl FrameConverter for VideoToolboxConverter { } let input_buffer = self.create_input_pixel_buffer(&input)?; - let output_buffer = self.create_output_pixel_buffer()?; - - let status = { - let session_guard = self.session.lock(); - unsafe { - VTPixelTransferSessionTransferImage(session_guard.0, input_buffer, output_buffer) - } - }; + let mut output_buffer = self.create_output_pixel_buffer()?; - unsafe { - CVPixelBufferRelease(input_buffer); - } + let status = self + .session + .lock() + .0 + .transfer(&input_buffer, &output_buffer); - if status != 0 { - unsafe { - CVPixelBufferRelease(output_buffer); - } + if let Err(error) = status { return Err(ConvertError::ConversionFailed(format!( - "VTPixelTransferSessionTransferImage failed: {status}" + "VTPixelTransferSessionTransferImage: {error}" ))); } @@ -345,13 +291,9 @@ impl FrameConverter for VideoToolboxConverter { ); } - self.copy_output_to_frame(output_buffer, output)?; + self.copy_output_to_frame(&mut output_buffer, output)?; output.set_pts(input.pts()); - unsafe { - CVPixelBufferRelease(output_buffer); - } - Ok(()) } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index e70151f06b8..32c26e1dc0b 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -1172,40 +1172,46 @@ async fn setup_camera( .senders .load(std::sync::atomic::Ordering::Relaxed) > 0 - && let Ok(bytes) = frame.native().bytes() { use cap_mediafoundation_utils::IMFMediaBufferExt; use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; - let data_len = bytes.len(); - if let Ok(buffer) = unsafe { MFCreateMemoryBuffer(data_len as u32) } { - let buffer_ready = { - if let Ok(mut lock) = buffer.lock_for_write() { - lock.copy_from_slice(&bytes); - true - } else { - false - } - }; - - if buffer_ready { - let _ = unsafe { buffer.SetCurrentLength(data_len as u32) }; - - #[allow(clippy::arc_with_non_send_sync)] - let buffer = std::sync::Arc::new(std::sync::Mutex::new(buffer)); - hand_native_frame_to_feed( - &native_recipient, - &native_ingress, - NativeCameraFrame { - buffer, - pixel_format: frame.native().pixel_format, - width: frame.native().width as u32, - height: frame.native().height as u32, - is_bottom_up: frame.native().is_bottom_up, - timestamp, - }, - ); - } + let buffer = if let Some(buffer) = frame.native().media_buffer() { + #[allow(clippy::arc_with_non_send_sync)] + Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + } else if let Ok(bytes) = frame.native().bytes() { + let data_len = bytes.len(); + unsafe { MFCreateMemoryBuffer(data_len as u32) } + .ok() + .and_then(|buffer| { + buffer + .lock_for_write() + .ok() + .map(|mut lock| lock.copy_from_slice(&bytes)) + .map(|()| buffer) + }) + .and_then(|buffer| { + unsafe { buffer.SetCurrentLength(data_len as u32) }.ok()?; + #[allow(clippy::arc_with_non_send_sync)] + Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + }) + } else { + None + }; + + if let Some(buffer) = buffer { + hand_native_frame_to_feed( + &native_recipient, + &native_ingress, + NativeCameraFrame { + buffer, + pixel_format: frame.native().pixel_format, + width: frame.native().width as u32, + height: frame.native().height as u32, + is_bottom_up: frame.native().is_bottom_up, + timestamp, + }, + ); } } diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 8d2fc226df9..74c95835a06 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -524,7 +524,50 @@ impl Muxer for WindowsMuxer { #[cfg(test)] mod tests { - use super::reference_video_frame; + use super::{MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, reference_video_frame}; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_timestamp::Timestamp; + use std::sync::{Arc, Mutex}; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + #[test] + fn native_nv12_frame_converts_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 13) as u8; + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let base_ptr = { + let lock = buffer.lock().expect("read lock"); + lock.as_ptr() + }; + + #[allow(clippy::arc_with_non_send_sync)] + let frame = NativeCameraFrame { + buffer: Arc::new(Mutex::new(buffer)), + pixel_format: PixelFormat::NV12, + width: width as u32, + height: height as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(std::time::Instant::now()), + }; + let mut decoder = MjpegDecoder::default(); + let converted = camera_frame_to_ffmpeg(&frame, &mut decoder).expect("conversion"); + + assert_eq!(converted.format(), ffmpeg::format::Pixel::NV12); + assert_eq!(converted.data(0).as_ptr(), base_ptr); + } #[test] fn retained_software_frame_shares_pixels_and_survives_original() { @@ -1375,6 +1418,21 @@ pub(crate) fn camera_frame_to_ffmpeg( return decode_mjpeg_frame(frame, mjpeg_decoder); } + if frame.pixel_format != cap_camera_windows::PixelFormat::UYVY422 + && frame.pixel_format != cap_camera_windows::PixelFormat::NV21 + && let Ok(buffer) = frame.buffer.lock() + && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( + &buffer, + frame.pixel_format, + frame.width as usize, + frame.height as usize, + frame.is_bottom_up, + ) + { + cap_camera_ffmpeg::CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + let ffmpeg_format = match frame.pixel_format { cap_camera_windows::PixelFormat::NV12 => ffmpeg::format::Pixel::NV12, cap_camera_windows::PixelFormat::YUYV422 => ffmpeg::format::Pixel::YUYV422, From 39434b5a936f5988ddbba4d10e96e3a904ef43af Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 00:49:30 +0100 Subject: [PATCH 2/4] fix(camera): align capture lock paths and preserve preview conversions Use the same Media Foundation lock type for every consumer of a capture buffer, including the hardware upload and preview copy paths, and cache the last converted frame for paced repeats. Revert the macOS camera-ffmpeg zero-copy wrapper so preview and ffmpeg consumers copy instead of holding CVPixelBuffer locks for an AVFrame lifetime. Fix the VideoToolbox pool attribute compile error, share DirectShow media types across the callback worker, initialize that worker as MTA, and expose callback drop counters. Restore NV21 to NV12 conversion in as_ffmpeg and log conversion statistics periodically. --- crates/camera-directshow/src/lib.rs | 76 +++++-- .../examples/frame-copy-bench.rs | 17 +- crates/camera-ffmpeg/src/lib.rs | 15 ++ crates/camera-ffmpeg/src/macos.rs | 199 +---------------- crates/camera-ffmpeg/src/windows.rs | 203 ++++++++++++++++-- crates/camera-mediafoundation/src/lib.rs | 8 +- crates/camera-windows/src/lib.rs | 7 + crates/camera/src/lib.rs | 28 +++ crates/camera/src/windows.rs | 4 + crates/frame-converter/src/videotoolbox.rs | 5 +- crates/recording/src/output_pipeline/win.rs | 63 ++++-- .../src/output_pipeline/win_fragmented_m4s.rs | 126 ++++++----- 12 files changed, 443 insertions(+), 308 deletions(-) diff --git a/crates/camera-directshow/src/lib.rs b/crates/camera-directshow/src/lib.rs index 3fb74fa08ad..81043f845de 100644 --- a/crates/camera-directshow/src/lib.rs +++ b/crates/camera-directshow/src/lib.rs @@ -39,6 +39,10 @@ pub fn initialize_directshow() -> windows_core::Result<()> { unsafe { CoInitialize(None) }.ok() } +fn initialize_directshow_worker() { + let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; +} + pub trait IPinExt { /// # Safety /// Do it correctly @@ -465,8 +469,10 @@ impl VideoInputDevice { .SetFormat(&**format) .map_err(StartCapturingError::Other)?; - let (sink_filter, callback_worker) = SinkFilter::new(format.clone(), callback) - .map_err(StartCapturingError::CallbackWorker)?; + let dropped = Arc::new(AtomicU64::new(0)); + let (sink_filter, callback_worker) = + SinkFilter::new(format.clone(), callback, dropped.clone()) + .map_err(StartCapturingError::CallbackWorker)?; let input_sink_pin = sink_filter .get_pin(0) @@ -522,6 +528,7 @@ impl VideoInputDevice { output_capture_pin: bound.output_pin, input_sink_pin, callback_worker, + dropped_samples: dropped, }) } } @@ -533,9 +540,14 @@ pub struct CaptureHandle { output_capture_pin: IPin, input_sink_pin: IPin, callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + // Chromium: VideoCaptureDeviceWin::StopAndDeallocate pub fn stop_capturing(self) -> windows_core::Result<()> { unsafe { self.media_control.Stop() }?; @@ -594,6 +606,11 @@ impl Iterator for VideoMediaTypesIterator<'_> { #[derive(Default, Debug)] pub struct AMMediaType(AM_MEDIA_TYPE); +// The media type owns its format block and is immutable after construction; +// the callback worker only reads it. +unsafe impl Send for AMMediaType {} +unsafe impl Sync for AMMediaType {} + impl AMMediaType { pub fn new(typ: &AM_MEDIA_TYPE) -> Self { Self(unsafe { copy_media_type(typ) }) @@ -605,6 +622,32 @@ impl AMMediaType { std::mem::forget(wrapper); inner } + + fn matches_media_type(&self, other: &AM_MEDIA_TYPE) -> bool { + let a = &self.0; + let b = other; + + if a.majortype != b.majortype + || a.subtype != b.subtype + || a.formattype != b.formattype + || a.cbFormat != b.cbFormat + { + return false; + } + + if a.cbFormat == 0 { + return true; + } + + if a.pbFormat.is_null() || b.pbFormat.is_null() { + return a.pbFormat == b.pbFormat; + } + + unsafe { + std::slice::from_raw_parts(a.pbFormat, a.cbFormat as usize) + == std::slice::from_raw_parts(b.pbFormat, b.cbFormat as usize) + } + } } impl Deref for AMMediaType { @@ -643,9 +686,9 @@ impl SinkFilter { pub(crate) fn new( desired_media_type: AMMediaType, callback: SinkCallback, + dropped: Arc, ) -> std::io::Result<(ComObject, CallbackWorker)> { let (callback_tx, callback_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); - let dropped = Arc::new(AtomicU64::new(0)); let callback_worker = CallbackWorker::spawn(callback, callback_rx, Arc::new(AtomicBool::new(false)))?; @@ -841,7 +884,7 @@ const CALLBACK_QUEUE_DEPTH: usize = 4; struct OwnedCallbackData { sample: IMediaSample, - media_type: AMMediaType, + media_type: Arc, timestamp: Duration, perf_counter: i64, } @@ -863,12 +906,12 @@ impl CallbackWorker { let handle = thread::Builder::new() .name("dshow-camera-callback".to_string()) .spawn(move || { - let _ = initialize_directshow(); + initialize_directshow_worker(); while !worker_stop.load(Ordering::Acquire) { match samples.recv_timeout(Duration::from_millis(100)) { Ok(data) => callback(CallbackData { sample: &data.sample, - media_type: &data.media_type, + media_type: data.media_type.as_ref(), timestamp: data.timestamp, perf_counter: data.perf_counter, }), @@ -897,8 +940,7 @@ impl Drop for CallbackWorker { #[implement(IPin, IMemInputPin)] struct SinkInputPin { desired_media_type: AMMediaType, - #[allow(unused)] - current_media_type: RefCell, + current_media_type: RefCell>, connected_pin: RefCell>, owner: RefCell>, callback_tx: SyncSender, @@ -969,7 +1011,7 @@ impl IPin_Impl for SinkInputPin_Impl { }; *connected_pin = Some(pconnector.clone()); self.current_media_type - .replace(AMMediaType::new(unsafe { &*pmt })); + .replace(Arc::new(AMMediaType::new(unsafe { &*pmt }))); S_OK.ok() } @@ -996,7 +1038,7 @@ impl IPin_Impl for SinkInputPin_Impl { .as_ref() .ok_or(VFW_E_NOT_CONNECTED)?; - unsafe { *pmt = self.current_media_type.borrow().clone().into_inner() }; + unsafe { *pmt = (**self.current_media_type.borrow()).clone().into_inner() }; Ok(()) } @@ -1099,16 +1141,16 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { return Ok(()); }; - unsafe { - if let Ok(new_media_type) = psample.GetMediaType() + let media_type = { + let mut current = self.current_media_type.borrow_mut(); + if let Ok(new_media_type) = unsafe { psample.GetMediaType() } && !new_media_type.is_null() + && !current.matches_media_type(unsafe { &*new_media_type }) { - self.current_media_type - .replace(AMMediaType::new(&*new_media_type)); + *current = Arc::new(AMMediaType::new(unsafe { &*new_media_type })); } - } - - let media_type = self.current_media_type.borrow().clone(); + Arc::clone(&*current) + }; let length = unsafe { psample.GetActualDataLength() }; diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs index 9e9fe178b0d..cf090310cef 100644 --- a/crates/camera-ffmpeg/examples/frame-copy-bench.rs +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -1,7 +1,7 @@ #[cfg(windows)] fn main() { use cap_camera::CapturedFrame; - use cap_camera_ffmpeg::{CapturedFrameExt, conversion_stats}; + use cap_camera_ffmpeg::media_buffer_as_ffmpeg; use cap_camera_windows::{Frame, PixelFormat}; use cap_mediafoundation_utils::IMFMediaBufferExt; use ffmpeg::{format::Pixel, frame::Video}; @@ -33,14 +33,18 @@ fn main() { ); let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); + let native_buffer = captured.native().media_buffer().expect("native buffer"); + for _ in 0..50 { - let frame = captured.as_ffmpeg().expect("warmup conversion"); + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("warmup conversion"); std::hint::black_box(frame); } let start = Instant::now(); for _ in 0..ITERATIONS { - let frame = captured.as_ffmpeg().expect("conversion"); + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("conversion"); std::hint::black_box(frame); } let zero_copy = start.elapsed(); @@ -78,7 +82,6 @@ fn main() { } let bytes_copy = start.elapsed(); - let stats = conversion_stats(); println!( "zero-copy: {:.3} ms/frame ({:.1} fps)", zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, @@ -97,12 +100,6 @@ fn main() { "bytes copy: {:.3} ms/frame", bytes_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64 ); - println!( - "stats: zero_copy={} copied={} avg_copy={:.3} us", - stats.zero_copy_frames, - stats.copied_frames, - stats.average_copy_micros() - ); } #[cfg(not(windows))] diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index c13e638281a..5905200a2b5 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -42,11 +42,26 @@ pub struct ConversionStats { impl ConversionStats { pub fn record_zero_copy(&self) { self.zero_copy_frames.fetch_add(1, Ordering::Relaxed); + self.log_periodically(); } pub fn record_copy(&self, elapsed_nanos: u64) { self.copied_frames.fetch_add(1, Ordering::Relaxed); self.copy_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); + self.log_periodically(); + } + + fn log_periodically(&self) { + let snapshot = self.snapshot(); + let total = snapshot.zero_copy_frames + snapshot.copied_frames; + if total > 0 && total.is_multiple_of(300) { + tracing::debug!( + zero_copy_frames = snapshot.zero_copy_frames, + copied_frames = snapshot.copied_frames, + average_copy_micros = snapshot.average_copy_micros(), + "Camera frame conversion stats" + ); + } } pub fn snapshot(&self) -> ConversionStatsSnapshot { diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index 81f91919c42..eb822a1efc0 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -1,29 +1,14 @@ use cap_camera::CapturedFrame; use cap_camera_avfoundation::ImageBufExt; use cidre::*; -use ffmpeg::{ffi, format::Pixel, software::scaling}; +use ffmpeg::{format::Pixel, software::scaling}; use std::{ cell::RefCell, - ffi::c_void, - ptr, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, + sync::atomic::{AtomicBool, Ordering}, }; use crate::{CONVERSION_STATS, CapturedFrameExt}; -type CVPixelBufferRef = *mut c_void; - -#[link(name = "CoreVideo", kind = "framework")] -unsafe extern "C" { - fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; - fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetWidth(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; -} - type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video); #[derive(thiserror::Error, Debug)] @@ -73,181 +58,6 @@ fn fourcc_to_pixel_format(fourcc: &str) -> Option { static FALLBACK_WARNING_LOGGED: AtomicBool = AtomicBool::new(false); -#[link(name = "CoreVideo", kind = "framework")] -unsafe extern "C" { - fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, flags: u64) -> i32; -} - -const CV_PIXEL_BUFFER_LOCK_READ_ONLY: u64 = 1; - -#[derive(Clone, Copy)] -struct SamplePlane { - ptr: *mut u8, - len: usize, - stride: usize, -} - -struct WrappedSampleBuf { - _image_buf: cidre::arc::R, - pixel_buffer: CVPixelBufferRef, - locked: bool, -} - -unsafe impl Send for WrappedSampleBuf {} -unsafe impl Sync for WrappedSampleBuf {} - -impl Drop for WrappedSampleBuf { - fn drop(&mut self) { - if self.locked { - unsafe { - CVPixelBufferUnlockBaseAddress(self.pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); - } - } - } -} - -unsafe extern "C" fn release_wrapped_sample_buf(opaque: *mut c_void, _data: *mut u8) { - if !opaque.is_null() { - drop(unsafe { Box::from_raw(opaque as *mut Arc) }); - } -} - -fn sample_buf_planes( - image_buf: &cidre::cv::ImageBuf, - format: &str, -) -> Option<(Pixel, Vec, Option)> { - let (pixel, plane_count, range) = match format { - "2vuy" => (Pixel::UYVY422, 0, None), - "yuvs" => (Pixel::YUYV422, 0, None), - "BGRA" => (Pixel::BGRA, 0, None), - "ARGB" => (Pixel::ARGB, 0, None), - "24BG" => (Pixel::BGR24, 0, None), - "24RG" => (Pixel::RGB24, 0, None), - "L008" | "GRAY" => (Pixel::GRAY8, 0, None), - "L016" => (Pixel::GRAY16LE, 0, None), - "420v" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::MPEG)), - "420f" => (Pixel::NV12, 2, Some(ffmpeg::color::Range::JPEG)), - "y420" => (Pixel::YUV420P, 3, None), - _ => return None, - }; - - if image_buf.plane_count() != plane_count { - return None; - } - - let mut planes = Vec::with_capacity(plane_count.max(1)); - if plane_count == 0 { - let pixel_buffer = image_buf as *const cidre::cv::ImageBuf as CVPixelBufferRef; - let ptr = unsafe { CVPixelBufferGetBaseAddress(pixel_buffer) }; - let stride = unsafe { CVPixelBufferGetBytesPerRow(pixel_buffer) }; - let height = unsafe { CVPixelBufferGetHeight(pixel_buffer) }; - if ptr.is_null() || stride == 0 || height == 0 { - return None; - } - planes.push(SamplePlane { - ptr, - len: stride.checked_mul(height)?, - stride, - }); - } else { - for plane in 0..plane_count { - let ptr = image_buf.plane_base_address(plane) as *mut u8; - let stride = image_buf.plane_bytes_per_row(plane); - let height = image_buf.plane_height(plane); - if ptr.is_null() || stride == 0 || height == 0 { - return None; - } - planes.push(SamplePlane { - ptr, - len: stride.checked_mul(height)?, - stride, - }); - } - } - - Some((pixel, planes, range)) -} - -fn wrap_sample_buf(sample_buf: &cidre::cm::SampleBuf) -> Option { - let image_buf = sample_buf.image_buf()?.retained(); - let format_desc = sample_buf.format_desc()?; - let format = cidre::four_cc_to_str(&mut format_desc.media_sub_type().to_be_bytes()); - let pixel_buffer = (&*image_buf) as *const cidre::cv::ImageBuf as CVPixelBufferRef; - - if unsafe { CVPixelBufferLockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY) } != 0 { - return None; - } - - let Some((pixel, planes, range)) = sample_buf_planes(&image_buf, format) else { - unsafe { - CVPixelBufferUnlockBaseAddress(pixel_buffer, CV_PIXEL_BUFFER_LOCK_READ_ONLY); - } - return None; - }; - - let owner = Arc::new(WrappedSampleBuf { - _image_buf: image_buf, - pixel_buffer, - locked: true, - }); - - let mut frame = ffmpeg::frame::Video::empty(); - let ptr = unsafe { frame.as_mut_ptr() }; - let format: ffi::AVPixelFormat = pixel.into(); - unsafe { - (*ptr).format = format as i32; - (*ptr).width = image_buf_width(&owner) as i32; - (*ptr).height = image_buf_height(&owner) as i32; - } - - let mut buffers = Vec::with_capacity(planes.len()); - for plane in &planes { - let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; - let buffer = unsafe { - ffi::av_buffer_create( - plane.ptr, - plane.len, - Some(release_wrapped_sample_buf), - opaque, - ffi::AV_BUFFER_FLAG_READONLY, - ) - }; - if buffer.is_null() { - unsafe { - drop(Box::from_raw(opaque as *mut Arc)); - for mut buffer in buffers { - ffi::av_buffer_unref(&mut buffer); - } - } - return None; - } - buffers.push(buffer); - } - - for (index, (buffer, plane)) in buffers.into_iter().zip(&planes).enumerate() { - unsafe { - (*ptr).buf[index] = buffer; - (*ptr).data[index] = plane.ptr; - (*ptr).linesize[index] = plane.stride as i32; - } - } - - if let Some(range) = range { - frame.set_color_range(range); - } - - Some(frame) -} - -fn image_buf_width(owner: &WrappedSampleBuf) -> usize { - unsafe { CVPixelBufferGetWidth(owner.pixel_buffer) } -} - -fn image_buf_height(owner: &WrappedSampleBuf) -> usize { - unsafe { CVPixelBufferGetHeight(owner.pixel_buffer) } -} - impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { sample_buf_as_ffmpeg(self.native().sample_buf()) @@ -259,11 +69,6 @@ impl CapturedFrameExt for CapturedFrame { pub fn sample_buf_as_ffmpeg( sample_buf: &cm::SampleBuf, ) -> Result { - if let Some(frame) = wrap_sample_buf(sample_buf) { - CONVERSION_STATS.record_zero_copy(); - return Ok(frame); - } - let started = std::time::Instant::now(); let frame = { let mut image_buf = sample_buf diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 34efbfcf89c..986a303d5c7 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -372,9 +372,7 @@ fn lock_mf_buffer( format: PixelFormat, width: usize, height: usize, -) -> Option<(WrappedMfBuffer, Vec, Pixel)> { - let pixel = mf_pixel_format(format)?; - +) -> Option<(WrappedMfBuffer, Vec)> { unsafe { if let Ok(buffer_2d) = buffer.cast::() { let mut scanline = ptr::null_mut(); @@ -403,7 +401,6 @@ fn lock_mf_buffer( lock: MfLock::TwoD2(buffer_2d), }, planes, - pixel, )); } @@ -433,7 +430,6 @@ fn lock_mf_buffer( lock: MfLock::TwoD(buffer_2d), }, planes, - pixel, )); } @@ -456,7 +452,6 @@ fn lock_mf_buffer( lock: MfLock::Contiguous, }, planes, - pixel, )); } @@ -474,14 +469,77 @@ pub fn media_buffer_as_ffmpeg( height: usize, is_bottom_up: bool, ) -> Option { - if is_bottom_up || width == 0 || height == 0 { + if !should_use_2d_lock(format, is_bottom_up) || width == 0 || height == 0 { return None; } - let (owner, planes, pixel) = lock_mf_buffer(buffer, format, width, height)?; + let pixel = mf_pixel_format(format)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) } +// UYVY and NV21 use the format-converting copy paths, which lock with +// IMFMediaBuffer::Lock; all consumers of a buffer must use the same lock type. +pub fn should_use_2d_lock(format: PixelFormat, is_bottom_up: bool) -> bool { + !is_bottom_up + && format != PixelFormat::UYVY422 + && format != PixelFormat::NV21 + && mf_pixel_format(format).is_some() +} + +pub fn with_media_buffer_data( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + f: impl FnOnce(&[u8], usize, usize) -> R, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let data = unsafe { std::slice::from_raw_parts(planes[0].ptr, total_len) }; + let primary_stride = planes[0].stride; + let secondary_stride = planes.get(1).map_or(primary_stride, |plane| plane.stride); + let result = f(data, primary_stride, secondary_stride); + drop(owner); + Some(result) +} + +pub fn copy_media_buffer_to_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let pixel = mf_pixel_format(format)?; + let row_bytes = mf_row_bytes(format, width)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let mut frame = FFVideo::new(pixel, width as u32, height as u32); + + for (index, plane) in planes.iter().enumerate() { + let plane_height = plane.len / plane.stride; + let destination_stride = frame.stride(index); + let destination = frame.data_mut(index); + + for row in 0..plane_height { + let source = + unsafe { std::slice::from_raw_parts(plane.ptr.add(row * plane.stride), row_bytes) }; + let destination_start = row * destination_stride; + destination[destination_start..destination_start + row_bytes].copy_from_slice(source); + } + } + + drop(owner); + Some(frame) +} + fn wrap_mf_frame( planes: &[MfPlane], pixel: Pixel, @@ -546,11 +604,14 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } - if let Some(buffer) = native.media_buffer() + let started = std::time::Instant::now(); + + if should_use_2d_lock(native.pixel_format, is_bottom_up) + && let Some(buffer) = native.media_buffer() && let Some(frame) = - media_buffer_as_ffmpeg(&buffer, native.pixel_format, width, height, is_bottom_up) + copy_media_buffer_to_ffmpeg(&buffer, native.pixel_format, width, height) { - CONVERSION_STATS.record_zero_copy(); + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); return Ok(frame); } @@ -560,7 +621,6 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } - let started = std::time::Instant::now(); let frame = match native.pixel_format { PixelFormat::YUV420P => { let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32); @@ -825,10 +885,96 @@ impl CapturedFrameExt for CapturedFrame { } #[cfg(test)] -mod zero_copy_tests { +mod conversion_tests { use super::*; + use cap_camera_windows::Frame; use cap_mediafoundation_utils::IMFMediaBufferExt; - use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; + + #[test] + fn nv12_2d_buffer_can_be_wrapped_twice() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let first = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("first frame"); + let second = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + + assert!(second.is_some(), "second lock failed"); + drop(first); + let third = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + assert!(third.is_some(), "third lock failed"); + } + + #[test] + fn copy_conversion_can_nest_inside_zero_copy_frame() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let zero_copy = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("zero-copy frame"); + let copied = copy_media_buffer_to_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + ) + .expect("copied frame"); + + assert_eq!(copied.format(), Pixel::NV12); + assert_eq!(copied.width(), width); + assert_eq!(copied.height(), height); + + let data_len = with_media_buffer_data( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + |data, primary_stride, secondary_stride| { + assert_eq!(primary_stride, secondary_stride); + assert!(primary_stride >= width as usize); + data.len() + }, + ) + .expect("locked data"); + + assert!(data_len >= width as usize * height as usize * 3 / 2); + drop(zero_copy); + } #[test] fn nv12_media_buffer_is_wrapped_without_copying() { @@ -869,4 +1015,33 @@ mod zero_copy_tests { assert_eq!(frame.data(1)[0], 0); assert_eq!(frame.data(1)[stride + 2], 5); } + + #[test] + fn as_ffmpeg_keeps_nv12_for_nv21_frames() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = Frame::from_media_foundation( + buffer, + PixelFormat::NV21, + width, + height, + false, + std::time::Duration::ZERO, + 0, + ); + let captured = + cap_camera::CapturedFrame::from_windows_frame(frame, std::time::Duration::ZERO); + let converted = captured.as_ffmpeg().expect("conversion"); + + assert_eq!(converted.format(), Pixel::NV12); + assert_eq!(converted.width(), width as u32); + assert_eq!(converted.height(), height as u32); + } } diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index e09fa426e21..4378ef24b5c 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -264,7 +264,7 @@ impl Device { let video_callback = VideoCallback { event_tx, samples: sample_tx, - dropped, + dropped: dropped.clone(), } .into_object(); @@ -389,6 +389,7 @@ impl Device { engine: Some(engine), event_rx, callback_worker, + dropped_samples: dropped, }) } } @@ -425,6 +426,7 @@ pub struct CaptureHandle { engine: Option, event_rx: Receiver, callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { @@ -432,6 +434,10 @@ impl CaptureHandle { &self.event_rx } + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + pub fn stop_capturing(mut self) -> windows_core::Result<()> { let result = if let Some(engine) = self.engine.take() { unsafe { engine.StopPreview() } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index 526a279ffd6..c1badbd9e68 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -173,6 +173,13 @@ pub enum CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + match self { + CaptureHandle::MediaFoundation(handle) => handle.dropped_samples(), + CaptureHandle::DirectShow(handle) => handle.dropped_samples(), + } + } + pub fn stop_capturing(self) -> windows_core::Result<()> { match self { CaptureHandle::MediaFoundation(handle) => handle.stop_capturing(), diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index c476d78f5f9..e4e9591c032 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -283,7 +283,28 @@ pub struct CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + #[cfg(windows)] + { + self.native + .as_ref() + .map(NativeCaptureHandle::dropped_samples) + .unwrap_or(0) + } + #[cfg(not(windows))] + { + 0 + } + } + pub fn stop_capturing(mut self) -> Result<(), String> { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture stopped with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing()?; } @@ -293,6 +314,13 @@ impl CaptureHandle { impl Drop for CaptureHandle { fn drop(&mut self) { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture released with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing().ok(); } diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index 07d72fa184d..73c9264cec2 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -96,6 +96,10 @@ pub struct WindowsCaptureHandle { } impl WindowsCaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.inner.dropped_samples() + } + pub fn stop_capturing(self) -> Result<(), String> { self.inner.stop_capturing().map_err(|e| e.to_string()) } diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index e0a104cd95e..b16a3b65f42 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -76,10 +76,7 @@ fn create_pool( height.as_type_ref(), io_surface.as_type_ref(), ], - ) - .ok_or_else(|| { - ConvertError::HardwareUnavailable("VideoToolbox pixel buffer pool attributes".to_string()) - })?; + ); cv::PixelBufPool::new(None, Some(&**attrs)) .map(PoolHandle) diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 74c95835a06..d8bf8e06d5a 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -1418,8 +1418,7 @@ pub(crate) fn camera_frame_to_ffmpeg( return decode_mjpeg_frame(frame, mjpeg_decoder); } - if frame.pixel_format != cap_camera_windows::PixelFormat::UYVY422 - && frame.pixel_format != cap_camera_windows::PixelFormat::NV21 + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) && let Ok(buffer) = frame.buffer.lock() && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( &buffer, @@ -1885,25 +1884,45 @@ pub fn upload_mf_buffer_to_texture( buffers: &mut CameraBuffers, ) -> windows::core::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - use windows::Win32::Graphics::Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, - }; - use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; - - let dxgi_format = frame.dxgi_format(); let buffer_guard = frame .buffer .lock() .map_err(|_| windows::core::Error::from(windows::core::HRESULT(-1)))?; - let lock = buffer_guard.lock()?; - let original_data = &*lock; + let buffer = &*buffer_guard; + + // The zero-copy and preview paths lock these formats with Lock2D; the + // upload must use the same lock type to avoid MF_E_INVALIDREQUEST. + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) + && let Some(texture) = cap_camera_ffmpeg::with_media_buffer_data( + buffer, + frame.pixel_format, + frame.width as usize, + frame.height as usize, + |data, primary_stride, secondary_stride| { + upload_mf_buffer_data_to_texture( + device, + frame, + data, + CameraBufferLayout { + primary_stride, + secondary_stride, + }, + buffers, + ) + }, + ) + { + return texture; + } + + let lock = buffer.lock()?; + let data = &*lock; let layout = infer_camera_buffer_layout( frame.pixel_format, frame.width as usize, frame.height as usize, - original_data.len(), + data.len(), ) .ok_or_else(|| { windows::core::Error::new( @@ -1913,11 +1932,29 @@ pub fn upload_mf_buffer_to_texture( frame.pixel_format, frame.width, frame.height, - original_data.len() + data.len() ), ) })?; + upload_mf_buffer_data_to_texture(device, frame, data, layout, buffers) +} + +fn upload_mf_buffer_data_to_texture( + device: &ID3D11Device, + frame: &NativeCameraFrame, + original_data: &[u8], + layout: CameraBufferLayout, + buffers: &mut CameraBuffers, +) -> windows::core::Result { + use windows::Win32::Graphics::Direct3D11::{ + D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, + }; + use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; + + let dxgi_format = frame.dxgi_format(); + let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; let needs_flip = frame.is_bottom_up; let mut row_pitch = layout.primary_stride as u32; diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index 4e61d2300ed..97fbd0cf402 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -1100,7 +1100,9 @@ impl WindowsFragmentedM4SCameraMuxer { } let mut pacer = FramePacer::new(video_config.fps()); - let mut last_frame: Option = None; + // Repeats share the last converted frame so the capture buffer + // is locked once instead of on every paced repeat. + let mut last_ffmpeg_frame: Option = None; let mut first_timestamp: Option = None; let mut slow_convert_count = 0u32; @@ -1122,17 +1124,14 @@ impl WindowsFragmentedM4SCameraMuxer { } }; - let encode_camera_frame_fn = + let convert_camera_frame = |camera_frame: &NativeCameraFrame, - timestamp: Duration, slow_convert_count: &mut u32, - slow_encode_count: &mut u32, - total_frames: &mut u64, - encoder: &Arc>, mjpeg_decoder: &mut MjpegDecoder| - -> anyhow::Result<()> { + -> anyhow::Result { let convert_start = std::time::Instant::now(); - let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); + let ffmpeg_frame_result = + camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); let convert_elapsed_ms = convert_start.elapsed().as_millis(); if convert_elapsed_ms > SLOW_THRESHOLD_MS { @@ -1147,41 +1146,42 @@ impl WindowsFragmentedM4SCameraMuxer { } } - match ffmpeg_frame_result { - Ok(ffmpeg_frame) => { - let encode_start = std::time::Instant::now(); + ffmpeg_frame_result + }; - match encoder.lock() { - Ok(mut enc) => { - if let Err(e) = enc.queue_frame(ffmpeg_frame, timestamp) { - warn!("Failed to encode camera frame: {e}"); - } - } - Err(_) => { - return Err(anyhow!( - "Camera encoder mutex poisoned - all subsequent frames would be lost" - )); - } - } + let encode_camera_frame_fn = + |ffmpeg_frame: ffmpeg::frame::Video, + timestamp: Duration, + slow_encode_count: &mut u32, + total_frames: &mut u64, + encoder: &Arc>| + -> anyhow::Result<()> { + let encode_start = std::time::Instant::now(); - let encode_elapsed_ms = encode_start.elapsed().as_millis(); - - if encode_elapsed_ms > SLOW_THRESHOLD_MS { - *slow_encode_count += 1; - if *slow_encode_count <= 5 - || slow_encode_count.is_multiple_of(100) - { - debug!( - elapsed_ms = encode_elapsed_ms, - count = *slow_encode_count, - "Camera encoder.queue_frame exceeded {}ms threshold", - SLOW_THRESHOLD_MS - ); - } + match encoder.lock() { + Ok(mut enc) => { + if let Err(e) = enc.queue_frame(ffmpeg_frame, timestamp) { + warn!("Failed to encode camera frame: {e}"); } } - Err(e) => { - warn!("Failed to convert camera frame: {e:?}"); + Err(_) => { + return Err(anyhow!( + "Camera encoder mutex poisoned - all subsequent frames would be lost" + )); + } + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + *slow_encode_count += 1; + if *slow_encode_count <= 5 || slow_encode_count.is_multiple_of(100) { + debug!( + elapsed_ms = encode_elapsed_ms, + count = *slow_encode_count, + "Camera encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); } } @@ -1201,17 +1201,30 @@ impl WindowsFragmentedM4SCameraMuxer { break; } - let (camera_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { - Paced::Frame(frame, ts) => { - last_frame = Some(frame.clone()); - (frame, ts) - } + let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, ts) => match convert_camera_frame( + &frame, + &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => { + last_ffmpeg_frame = Some(reference_video_frame(&ffmpeg_frame)); + (ffmpeg_frame, ts) + } + Err(e) => { + warn!("Failed to convert camera frame: {e:?}"); + let Some(last) = last_ffmpeg_frame.as_ref() else { + continue; + }; + (reference_video_frame(last), ts) + } + }, Paced::Repeat(ts) => { - let Some(frame) = &last_frame else { + let Some(last) = last_ffmpeg_frame.as_ref() else { continue; }; duplicated_frames += 1; - (frame.clone(), ts) + (reference_video_frame(last), ts) } Paced::End => { let mut drained = 0u64; @@ -1223,14 +1236,25 @@ impl WindowsFragmentedM4SCameraMuxer { Ok(Some((frame, ts))) => { let normalized_ts = normalize_timestamp(ts, &mut first_timestamp); - if let Err(e) = encode_camera_frame_fn( + let ffmpeg_frame = match convert_camera_frame( &frame, - normalized_ts, &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => ffmpeg_frame, + Err(e) => { + warn!( + "Failed to convert drained camera frame: {e:?}" + ); + continue; + } + }; + if let Err(e) = encode_camera_frame_fn( + ffmpeg_frame, + normalized_ts, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { warn!("Failed to encode drained camera frame: {e}"); break; @@ -1259,13 +1283,11 @@ impl WindowsFragmentedM4SCameraMuxer { let normalized_ts = normalize_timestamp(timestamp, &mut first_timestamp); if let Err(e) = encode_camera_frame_fn( - &camera_frame, + ffmpeg_frame, normalized_ts, - &mut slow_convert_count, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { error!("Camera encoder mutex poisoned - encoder thread likely panicked, stopping"); return Err(e); From 891585c86fd7eb17ff12dc772c504f972f3375d5 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 05:41:46 +0100 Subject: [PATCH 3/4] perf(camera): remove per-frame allocations and GPU copies on Windows Wrap each MF buffer in a single AVBufferRef with stack plane metadata, cutting the zero-copy path from 0.002 to 0.001 ms/frame. Replace the per-frame Arc> with a direct COM-backed CameraBuffer. Cache the D3D11 upload texture and immediate context and use UpdateSubresource instead of CreateTexture2D per frame: 0.733 vs 0.956 ms/frame for 1080p NV12. Upload straight into the hardware encoder's video-processor input texture to skip the extra CopyResource, while still running VideoProcessorBlt for conversion and scaling. Add camera-upload and zero-copy/preview-copy benchmarks. --- .../examples/frame-copy-bench.rs | 77 ++++++-- crates/camera-ffmpeg/src/windows.rs | 121 ++++++------ crates/enc-mediafoundation/src/video/h264.rs | 11 +- .../src/video/video_processor.rs | 27 ++- .../recording/examples/camera-upload-bench.rs | 130 +++++++++++++ crates/recording/src/feeds/camera.rs | 8 +- crates/recording/src/lib.rs | 2 + crates/recording/src/output_pipeline/win.rs | 181 ++++++++++++------ crates/verify/tests/recording.rs | 6 +- 9 files changed, 418 insertions(+), 145 deletions(-) create mode 100644 crates/recording/examples/camera-upload-bench.rs diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs index cf090310cef..ef56f6f12da 100644 --- a/crates/camera-ffmpeg/examples/frame-copy-bench.rs +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -1,29 +1,36 @@ #[cfg(windows)] fn main() { use cap_camera::CapturedFrame; - use cap_camera_ffmpeg::media_buffer_as_ffmpeg; + use cap_camera_ffmpeg::{copy_media_buffer_to_ffmpeg, media_buffer_as_ffmpeg}; use cap_camera_windows::{Frame, PixelFormat}; use cap_mediafoundation_utils::IMFMediaBufferExt; use ffmpeg::{format::Pixel, frame::Video}; use std::time::{Duration, Instant}; - use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; const WIDTH: usize = 1920; const HEIGHT: usize = 1080; const ITERATIONS: usize = 1000; let len = WIDTH * HEIGHT * 3 / 2; - let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + + let surface = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + } + .expect("2D camera buffer"); { - let mut lock = buffer.lock_for_write().expect("buffer lock"); + let mut lock = surface.lock_for_write().expect("surface lock"); for (index, byte) in lock.iter_mut().enumerate() { *byte = (index * 31) as u8; } } - unsafe { buffer.SetCurrentLength(len as u32) }.expect("buffer length"); - - let frame = Frame::from_media_foundation( - buffer, + let surface_frame = Frame::from_media_foundation( + surface, PixelFormat::NV12, WIDTH, HEIGHT, @@ -31,10 +38,29 @@ fn main() { Duration::ZERO, 0, ); - let captured = CapturedFrame::from_windows_frame(frame, Duration::ZERO); - + let captured = CapturedFrame::from_windows_frame(surface_frame, Duration::ZERO); let native_buffer = captured.native().media_buffer().expect("native buffer"); + let memory = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + { + let mut lock = memory.lock_for_write().expect("buffer lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + unsafe { memory.SetCurrentLength(len as u32) }.expect("buffer length"); + let memory_frame = Frame::from_media_foundation( + memory, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let copy_captured = CapturedFrame::from_windows_frame(memory_frame, Duration::ZERO); + let copy_bytes = copy_captured.native().bytes().expect("camera bytes"); + for _ in 0..50 { let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) .expect("warmup conversion"); @@ -49,20 +75,34 @@ fn main() { } let zero_copy = start.elapsed(); - let bytes = captured.native().bytes().expect("camera bytes"); + for _ in 0..50 { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("warmup preview copy"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("preview copy"); + std::hint::black_box(frame); + } + let preview_copy = start.elapsed(); + let start = Instant::now(); for _ in 0..ITERATIONS { let mut frame = Video::new(Pixel::NV12, WIDTH as u32, HEIGHT as u32); let stride = frame.stride(0); for row in 0..HEIGHT { frame.data_mut(0)[row * stride..row * stride + WIDTH] - .copy_from_slice(&bytes[row * WIDTH..row * WIDTH + WIDTH]); + .copy_from_slice(©_bytes[row * WIDTH..row * WIDTH + WIDTH]); } let uv_stride = frame.stride(1); let uv_offset = WIDTH * HEIGHT; for row in 0..HEIGHT / 2 { - frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH] - .copy_from_slice(&bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH]); + frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH].copy_from_slice( + ©_bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH], + ); } std::hint::black_box(frame); } @@ -83,12 +123,17 @@ fn main() { let bytes_copy = start.elapsed(); println!( - "zero-copy: {:.3} ms/frame ({:.1} fps)", + "zero-copy: {:.3} ms/frame ({:.1} fps)", zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / zero_copy.as_secs_f64() ); println!( - "copy: {:.3} ms/frame ({:.1} fps)", + "preview copy: {:.3} ms/frame ({:.1} fps)", + preview_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / preview_copy.as_secs_f64() + ); + println!( + "row copy: {:.3} ms/frame ({:.1} fps)", copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / copy.as_secs_f64() ); diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 986a303d5c7..95ba2cfc48e 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -1,4 +1,4 @@ -use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr, sync::Arc}; +use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr}; use cap_camera::CapturedFrame; use cap_camera_windows::PixelFormat; @@ -242,6 +242,35 @@ struct MfPlane { stride: usize, } +struct MfPlanes { + planes: [MfPlane; 3], + len: usize, +} + +impl MfPlanes { + fn new() -> Self { + Self { + planes: [MfPlane { + ptr: ptr::null_mut(), + len: 0, + stride: 0, + }; 3], + len: 0, + } + } + + fn push(&mut self, plane: MfPlane) { + if self.len < self.planes.len() { + self.planes[self.len] = plane; + self.len += 1; + } + } + + fn as_slice(&self) -> &[MfPlane] { + &self.planes[..self.len] + } +} + enum MfLock { TwoD2(IMF2DBuffer2), TwoD(IMF2DBuffer), @@ -276,7 +305,7 @@ impl Drop for WrappedMfBuffer { unsafe extern "C" fn release_wrapped_mf_buffer(opaque: *mut c_void, _data: *mut u8) { if !opaque.is_null() { - drop(unsafe { Box::from_raw(opaque as *mut Arc) }); + drop(unsafe { Box::from_raw(opaque as *mut WrappedMfBuffer) }); } } @@ -319,7 +348,7 @@ fn mf_planes( data: *mut u8, len: usize, stride: usize, -) -> Option> { +) -> Option { let row_bytes = mf_row_bytes(format, width)?; if stride < row_bytes || height == 0 { return None; @@ -327,27 +356,17 @@ fn mf_planes( let plane = |ptr: *mut u8, len: usize| MfPlane { ptr, len, stride }; let y_len = stride.checked_mul(height)?; + let mut planes = MfPlanes::new(); match format { - PixelFormat::NV12 | PixelFormat::NV21 => { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::P010 => { let uv_len = stride.checked_mul(height / 2)?; if len < y_len.checked_add(uv_len)? { return None; } - Some(vec![ - plane(data, y_len), - plane(unsafe { data.add(y_len) }, uv_len), - ]) - } - PixelFormat::P010 => { - let uv_len = stride.checked_mul(height / 2)?; - if len < y_len.checked_add(uv_len)? { - return None; - } - Some(vec![ - plane(data, y_len), - plane(unsafe { data.add(y_len) }, uv_len), - ]) + planes.push(plane(data, y_len)); + planes.push(plane(unsafe { data.add(y_len) }, uv_len)); + Some(planes) } PixelFormat::YUYV422 | PixelFormat::UYVY422 @@ -361,7 +380,8 @@ fn mf_planes( if len < y_len { return None; } - Some(vec![plane(data, y_len)]) + planes.push(plane(data, y_len)); + Some(planes) } PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => None, } @@ -372,7 +392,7 @@ fn lock_mf_buffer( format: PixelFormat, width: usize, height: usize, -) -> Option<(WrappedMfBuffer, Vec)> { +) -> Option<(WrappedMfBuffer, MfPlanes)> { unsafe { if let Ok(buffer_2d) = buffer.cast::() { let mut scanline = ptr::null_mut(); @@ -475,7 +495,7 @@ pub fn media_buffer_as_ffmpeg( let pixel = mf_pixel_format(format)?; let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; - wrap_mf_frame(&planes, pixel, width as u32, height as u32, Arc::new(owner)) + wrap_mf_frame(&planes, pixel, width as u32, height as u32, owner) } // UYVY and NV21 use the format-converting copy paths, which lock with @@ -499,6 +519,7 @@ pub fn with_media_buffer_data( } let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); let total_len = planes.iter().map(|plane| plane.len).sum::(); let data = unsafe { std::slice::from_raw_parts(planes[0].ptr, total_len) }; let primary_stride = planes[0].stride; @@ -521,6 +542,7 @@ pub fn copy_media_buffer_to_ffmpeg( let pixel = mf_pixel_format(format)?; let row_bytes = mf_row_bytes(format, width)?; let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); let mut frame = FFVideo::new(pixel, width as u32, height as u32); for (index, plane) in planes.iter().enumerate() { @@ -541,12 +563,32 @@ pub fn copy_media_buffer_to_ffmpeg( } fn wrap_mf_frame( - planes: &[MfPlane], + planes: &MfPlanes, pixel: Pixel, width: u32, height: u32, - owner: Arc, + owner: WrappedMfBuffer, ) -> Option { + let planes = planes.as_slice(); + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let opaque = Box::into_raw(Box::new(owner)) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + planes[0].ptr, + total_len, + Some(release_wrapped_mf_buffer), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut WrappedMfBuffer)); + } + return None; + } + let mut frame = FFVideo::empty(); let ptr = unsafe { frame.as_mut_ptr() }; let format: ffi::AVPixelFormat = pixel.into(); @@ -554,37 +596,8 @@ fn wrap_mf_frame( (*ptr).format = format as i32; (*ptr).width = width as i32; (*ptr).height = height as i32; - } - - let mut buffers = Vec::with_capacity(planes.len()); - for plane in planes { - let opaque = Box::into_raw(Box::new(Arc::clone(&owner))) as *mut c_void; - let buffer = unsafe { - ffi::av_buffer_create( - plane.ptr, - plane.len, - Some(release_wrapped_mf_buffer), - opaque, - ffi::AV_BUFFER_FLAG_READONLY, - ) - }; - - if buffer.is_null() { - unsafe { - drop(Box::from_raw(opaque as *mut Arc)); - for mut buffer in buffers { - ffi::av_buffer_unref(&mut buffer); - } - } - return None; - } - - buffers.push(buffer); - } - - for (index, (buffer, plane)) in buffers.into_iter().zip(planes).enumerate() { - unsafe { - (*ptr).buf[index] = buffer; + (*ptr).buf[0] = buffer; + for (index, plane) in planes.iter().enumerate() { (*ptr).data[index] = plane.ptr; (*ptr).linesize[index] = plane.stride as i32; } diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 9d34e047605..b7b69e3c757 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -240,6 +240,10 @@ impl From for EncoderRuntimeError { unsafe impl Send for H264Encoder {} impl H264Encoder { + pub fn input_texture(&self) -> &ID3D11Texture2D { + self.video_processor.input_texture() + } + #[allow(clippy::too_many_arguments)] fn new_with_scaled_output_with_flags( d3d_device: &ID3D11Device, @@ -710,7 +714,12 @@ impl H264Encoder { } let process_result = (|| -> windows::core::Result<()> { - self.video_processor.process_texture(&texture)?; + if texture.as_raw() != self.video_processor.input_texture().as_raw() + { + self.video_processor.process_texture(&texture)?; + } else { + self.video_processor.process_input_texture()?; + } let input_buffer = MFCreateDXGISurfaceBuffer( &ID3D11Texture2D::IID, self.video_processor.output_texture(), diff --git a/crates/enc-mediafoundation/src/video/video_processor.rs b/crates/enc-mediafoundation/src/video/video_processor.rs index fe26d73f337..147532a3796 100644 --- a/crates/enc-mediafoundation/src/video/video_processor.rs +++ b/crates/enc-mediafoundation/src/video/video_processor.rs @@ -218,6 +218,10 @@ impl VideoProcessor { &self.video_output_texture } + pub fn input_texture(&self) -> &ID3D11Texture2D { + &self.video_input_texture + } + pub fn process_texture( &mut self, input_texture: &ID3D11Texture2D, @@ -226,18 +230,23 @@ impl VideoProcessor { // texture that matches the input size we were initialized with. unsafe { - // Copy the texture to the video input texture self.d3d_context .CopyResource(&self.video_input_texture, input_texture); + } - // Convert to NV12 - let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { - Enable: true.into(), - OutputIndex: 0, - InputFrameOrField: 0, - pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), - ..Default::default() - }; + self.process_input_texture() + } + + pub fn process_input_texture(&mut self) -> windows::core::Result<()> { + let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + OutputIndex: 0, + InputFrameOrField: 0, + pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), + ..Default::default() + }; + + unsafe { self.video_context.VideoProcessorBlt( &self.video_processor, &self.video_output, diff --git a/crates/recording/examples/camera-upload-bench.rs b/crates/recording/examples/camera-upload-bench.rs new file mode 100644 index 00000000000..e58c3762746 --- /dev/null +++ b/crates/recording/examples/camera-upload-bench.rs @@ -0,0 +1,130 @@ +#[cfg(windows)] +fn main() -> anyhow::Result<()> { + use cap_camera_ffmpeg::with_media_buffer_data; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_recording::{ + CameraBuffer, CameraBuffers, NativeCameraFrame, create_d3d_device, + upload_mf_buffer_to_texture, + }; + use cap_timestamp::Timestamp; + use std::time::Instant; + use windows::Win32::{ + Graphics::{ + Direct3D11::{ + D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, + D3D11_USAGE_DEFAULT, + }, + Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, + }, + Media::MediaFoundation::MFCreate2DMediaBuffer, + }; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 500; + + cap_mediafoundation_utils::thread_init(); + let device = create_d3d_device()?; + let buffer = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + }?; + + { + let mut lock = buffer.lock_for_write()?; + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 17) as u8; + } + } + + let frame = NativeCameraFrame { + buffer: CameraBuffer::new(buffer), + pixel_format: PixelFormat::NV12, + width: WIDTH as u32, + height: HEIGHT as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(Instant::now()), + }; + let mut buffers = CameraBuffers::new(); + + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + let elapsed = start.elapsed(); + + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: WIDTH as u32, + Height: HEIGHT as u32, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + + let start = Instant::now(); + with_media_buffer_data( + frame.buffer.get(), + PixelFormat::NV12, + WIDTH, + HEIGHT, + |data, pitch, _| { + for _ in 0..ITERATIONS { + let subresource = D3D11_SUBRESOURCE_DATA { + pSysMem: data.as_ptr() as *const _, + SysMemPitch: pitch as u32, + SysMemSlicePitch: 0, + }; + let mut texture = None; + unsafe { + device.CreateTexture2D( + &texture_desc, + Some(&subresource), + Some(&mut texture), + )?; + } + std::hint::black_box(texture); + } + + Ok::<_, windows::core::Error>(()) + }, + ) + .expect("locked data")?; + let uncached = start.elapsed(); + + println!( + "cached upload: {:.3} ms/frame ({:.1} fps)", + elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / elapsed.as_secs_f64() + ); + println!( + "uncached upload: {:.3} ms/frame ({:.1} fps)", + uncached.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / uncached.as_secs_f64() + ); + + Ok(()) +} + +#[cfg(not(windows))] +fn main() { + println!("camera-upload-bench requires Windows"); +} diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 32c26e1dc0b..7e001a82743 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -25,7 +25,7 @@ use tokio::{sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use crate::ffmpeg::FFmpegVideoFrame; -use crate::output_pipeline::{NativeCameraFrame, VideoFrame}; +use crate::output_pipeline::{CameraBuffer, NativeCameraFrame, VideoFrame}; const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); /// Outer deadline for camera readiness. Must cover both capture attempts on @@ -1177,8 +1177,7 @@ async fn setup_camera( use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; let buffer = if let Some(buffer) = frame.native().media_buffer() { - #[allow(clippy::arc_with_non_send_sync)] - Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + Some(CameraBuffer::new(buffer)) } else if let Ok(bytes) = frame.native().bytes() { let data_len = bytes.len(); unsafe { MFCreateMemoryBuffer(data_len as u32) } @@ -1192,8 +1191,7 @@ async fn setup_camera( }) .and_then(|buffer| { unsafe { buffer.SetCurrentLength(data_len as u32) }.ok()?; - #[allow(clippy::arc_with_non_send_sync)] - Some(std::sync::Arc::new(std::sync::Mutex::new(buffer))) + Some(CameraBuffer::new(buffer)) }) } else { None diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index a7dcbbcc4c9..688a2939190 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -22,6 +22,8 @@ pub mod upload_preparation; pub mod upload_resume; pub mod upload_verification; +#[cfg(windows)] +pub use capture_pipeline::create_d3d_device; #[cfg(target_os = "linux")] pub use capture_pipeline::target_to_display_and_crop; pub use resolution_limits::{H264_MAX_DIMENSION, calculate_gpu_compatible_size}; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index d8bf8e06d5a..bc621e8826f 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -120,10 +120,13 @@ use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, Win32::Graphics::{ - Direct3D11::ID3D11Device, + Direct3D11::{ + D3D11_TEXTURE2D_DESC, ID3D11Device, ID3D11DeviceContext, ID3D11Resource, + ID3D11Texture2D, + }, Dxgi::Common::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_R8G8B8A8_UNORM, - DXGI_FORMAT_YUY2, + DXGI_FORMAT_UNKNOWN, DXGI_FORMAT_YUY2, }, }, }; @@ -524,11 +527,13 @@ impl Muxer for WindowsMuxer { #[cfg(test)] mod tests { - use super::{MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, reference_video_frame}; + use super::{ + CameraBuffer, MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, + reference_video_frame, + }; use cap_camera_windows::PixelFormat; use cap_mediafoundation_utils::IMFMediaBufferExt; use cap_timestamp::Timestamp; - use std::sync::{Arc, Mutex}; use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; #[test] @@ -553,9 +558,8 @@ mod tests { lock.as_ptr() }; - #[allow(clippy::arc_with_non_send_sync)] let frame = NativeCameraFrame { - buffer: Arc::new(Mutex::new(buffer)), + buffer: CameraBuffer::new(buffer), pixel_format: PixelFormat::NV12, width: width as u32, height: height as u32, @@ -662,10 +666,29 @@ pub(crate) fn duration_to_timespan(duration: Duration) -> TimeSpan { } } +type CameraMediaBuffer = windows::Win32::Media::MediaFoundation::IMFMediaBuffer; + +#[derive(Clone)] +pub struct CameraBuffer(CameraMediaBuffer); + +impl CameraBuffer { + pub fn new(buffer: CameraMediaBuffer) -> Self { + Self(buffer) + } + + pub fn get(&self) -> &CameraMediaBuffer { + &self.0 + } +} + +// Media Foundation buffers are free-threaded; pixel access is guarded by the +// per-frame 2D or contiguous lock held by the consumer. +unsafe impl Send for CameraBuffer {} +unsafe impl Sync for CameraBuffer {} + #[derive(Clone)] pub struct NativeCameraFrame { - pub buffer: - std::sync::Arc>, + pub buffer: CameraBuffer, pub pixel_format: cap_camera_windows::PixelFormat, pub width: u32, pub height: u32, @@ -939,7 +962,8 @@ impl Muxer for WindowsCameraMuxer { let mut last_frame: Option = None; let mut first_timestamp: Option = None; let mut frame_count = 0u64; - let mut camera_buffers = CameraBuffers::new(); + let mut camera_buffers = + CameraBuffers::with_texture(encoder.input_texture().clone()); let result = encoder.run( Arc::new(AtomicBool::default()), @@ -1159,6 +1183,11 @@ impl AudioMuxer for WindowsCameraMuxer { pub struct CameraBuffers { uyvy_buffer: Vec, flip_buffer: Vec, + texture: Option, + texture_width: u32, + texture_height: u32, + texture_format: DXGI_FORMAT, + context: Option, } impl CameraBuffers { @@ -1166,9 +1195,66 @@ impl CameraBuffers { Self { uyvy_buffer: Vec::new(), flip_buffer: Vec::new(), + texture: None, + texture_width: 0, + texture_height: 0, + texture_format: DXGI_FORMAT_UNKNOWN, + context: None, + } + } + + pub fn with_texture(texture: ID3D11Texture2D) -> Self { + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { texture.GetDesc(&mut desc) }; + + Self { + texture: Some(texture), + texture_width: desc.Width, + texture_height: desc.Height, + texture_format: desc.Format, + ..Self::new() } } + fn ensure_texture( + &mut self, + device: &ID3D11Device, + desc: &D3D11_TEXTURE2D_DESC, + ) -> windows::core::Result { + if self.texture.is_none() + || self.texture_width != desc.Width + || self.texture_height != desc.Height + || self.texture_format != desc.Format + { + let mut texture = None; + unsafe { device.CreateTexture2D(desc, None, Some(&mut texture))? }; + self.texture = texture; + self.texture_width = desc.Width; + self.texture_height = desc.Height; + self.texture_format = desc.Format; + } + + self.texture.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "CreateTexture2D succeeded but returned no texture", + ) + }) + } + + fn context(&mut self, device: &ID3D11Device) -> windows::core::Result { + if self.context.is_none() { + self.context = Some(unsafe { device.GetImmediateContext()? }); + } + + self.context.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "GetImmediateContext returned no context", + ) + }) + } + fn ensure_uyvy_capacity(&mut self, size: usize) -> &mut [u8] { if self.uyvy_buffer.len() < size { self.uyvy_buffer.resize(size, 0); @@ -1419,9 +1505,8 @@ pub(crate) fn camera_frame_to_ffmpeg( } if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) - && let Ok(buffer) = frame.buffer.lock() && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( - &buffer, + frame.buffer.get(), frame.pixel_format, frame.width as usize, frame.height as usize, @@ -1447,11 +1532,9 @@ pub(crate) fn camera_frame_to_ffmpeg( other => anyhow::bail!("Unsupported camera pixel format: {:?}", other), }; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1711,11 +1794,9 @@ fn decode_mjpeg_frame( ) -> anyhow::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1885,11 +1966,7 @@ pub fn upload_mf_buffer_to_texture( ) -> windows::core::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - let buffer_guard = frame - .buffer - .lock() - .map_err(|_| windows::core::Error::from(windows::core::HRESULT(-1)))?; - let buffer = &*buffer_guard; + let buffer = frame.buffer.get(); // The zero-copy and preview paths lock these formats with Lock2D; the // upload must use the same lock type to avoid MF_E_INVALIDREQUEST. @@ -1947,13 +2024,28 @@ fn upload_mf_buffer_data_to_texture( layout: CameraBufferLayout, buffers: &mut CameraBuffers, ) -> windows::core::Result { - use windows::Win32::Graphics::Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, - }; + use windows::Win32::Graphics::Direct3D11::{D3D11_BIND_SHADER_RESOURCE, D3D11_USAGE_DEFAULT}; use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; + use windows::core::Interface; let dxgi_format = frame.dxgi_format(); + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: frame.width, + Height: frame.height, + MipLevels: 1, + ArraySize: 1, + Format: dxgi_format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let texture: ID3D11Texture2D = buffers.ensure_texture(device, &texture_desc)?; + let context: ID3D11DeviceContext = buffers.context(device)?; let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; let needs_flip = frame.is_bottom_up; @@ -2039,36 +2131,11 @@ fn upload_mf_buffer_data_to_texture( } }; - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: frame.width, - Height: frame.height, - MipLevels: 1, - ArraySize: 1, - Format: dxgi_format, - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let subresource_data = D3D11_SUBRESOURCE_DATA { - pSysMem: data.as_ptr() as *const _, - SysMemPitch: row_pitch, - SysMemSlicePitch: 0, - }; + let resource = texture.cast::()?; unsafe { - let mut texture = None; - device.CreateTexture2D(&texture_desc, Some(&subresource_data), Some(&mut texture))?; - texture.ok_or_else(|| { - windows::core::Error::new( - windows::core::HRESULT(-1), - "CreateTexture2D succeeded but returned no texture", - ) - }) + context.UpdateSubresource(&resource, 0, None, data.as_ptr() as *const _, row_pitch, 0); } + + Ok(texture) } diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs index 61abf3a06b4..ffb3d26e8e7 100644 --- a/crates/verify/tests/recording.rs +++ b/crates/verify/tests/recording.rs @@ -31,6 +31,8 @@ use cap_project::{ InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, XY, }; +#[cfg(windows)] +use cap_recording::CameraBuffer; use cap_recording::{ NativeCameraFrame, RecordingDefaults, StudioQuality, feeds::{ @@ -1134,10 +1136,8 @@ fn camera_frame(slot: u32, capture: u64) -> Option { let buffer = unsafe { MFCreateMemoryBuffer(nv12.len() as u32) }.ok()?; buffer.lock_for_write().ok()?.copy_from_slice(&nv12); unsafe { buffer.SetCurrentLength(nv12.len() as u32) }.ok()?; - #[allow(clippy::arc_with_non_send_sync)] - let buffer = Arc::new(std::sync::Mutex::new(buffer)); Some(NativeCameraFrame { - buffer, + buffer: CameraBuffer::new(buffer), pixel_format: cap_camera_windows::PixelFormat::NV12, width: w as u32, height: h as u32, From 4c8420c388475f64ec202be25c2b07612d80c970 Mon Sep 17 00:00:00 2001 From: Juan Romero Date: Mon, 26 Oct 2026 07:40:28 +0100 Subject: [PATCH 4/4] fix(camera): restore non-Windows builds and benchmark direct upload Gate the Windows-only CameraBuffer import in feeds/camera.rs; the unconditional import broke macOS and Linux builds of cap-recording. Extend camera-upload-bench with the direct encoder-input path and a cached-upload-plus-CopyResource comparison. At 1080p NV12 the direct path is 1.053 ms/frame versus 1.359 ms/frame for the two-step path; cached and uncached uploads are 0.700 and 0.912 ms/frame. --- .../recording/examples/camera-upload-bench.rs | 53 ++++++++++++++++++- crates/recording/src/feeds/camera.rs | 4 +- 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/crates/recording/examples/camera-upload-bench.rs b/crates/recording/examples/camera-upload-bench.rs index e58c3762746..692d7858fb0 100644 --- a/crates/recording/examples/camera-upload-bench.rs +++ b/crates/recording/examples/camera-upload-bench.rs @@ -12,13 +12,14 @@ fn main() -> anyhow::Result<()> { use windows::Win32::{ Graphics::{ Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, + D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, ID3D11Resource, }, Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, }, Media::MediaFoundation::MFCreate2DMediaBuffer, }; + use windows::core::Interface; const WIDTH: usize = 1920; const HEIGHT: usize = 1080; @@ -110,6 +111,44 @@ fn main() -> anyhow::Result<()> { .expect("locked data")?; let uncached = start.elapsed(); + let encoder_desc = D3D11_TEXTURE2D_DESC { + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + ..texture_desc + }; + let mut encoder_texture = None; + unsafe { device.CreateTexture2D(&encoder_desc, None, Some(&mut encoder_texture))? }; + let encoder_texture = encoder_texture.expect("encoder input texture"); + let encoder_resource = encoder_texture.cast::()?; + let context = unsafe { device.GetImmediateContext()? }; + + let mut direct_buffers = CameraBuffers::with_texture(encoder_texture); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + let direct = start.elapsed(); + + let mut copy_buffers = CameraBuffers::new(); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + let cached_plus_copy = start.elapsed(); + println!( "cached upload: {:.3} ms/frame ({:.1} fps)", elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64, @@ -120,6 +159,16 @@ fn main() -> anyhow::Result<()> { uncached.as_secs_f64() * 1000.0 / ITERATIONS as f64, ITERATIONS as f64 / uncached.as_secs_f64() ); + println!( + "direct upload: {:.3} ms/frame ({:.1} fps)", + direct.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / direct.as_secs_f64() + ); + println!( + "cached + copy: {:.3} ms/frame ({:.1} fps)", + cached_plus_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / cached_plus_copy.as_secs_f64() + ); Ok(()) } diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index 7e001a82743..e8b3701cf44 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -25,7 +25,9 @@ use tokio::{sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use crate::ffmpeg::FFmpegVideoFrame; -use crate::output_pipeline::{CameraBuffer, NativeCameraFrame, VideoFrame}; +#[cfg(windows)] +use crate::output_pipeline::CameraBuffer; +use crate::output_pipeline::{NativeCameraFrame, VideoFrame}; const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); /// Outer deadline for camera readiness. Must cover both capture attempts on