diff --git a/Cargo.lock b/Cargo.lock index 5edf1df1581..6ff945d5749 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1421,6 +1421,7 @@ dependencies = [ "cap-camera", "cap-camera-avfoundation", "cap-camera-windows", + "cap-mediafoundation-utils", "cidre", "ffmpeg-next", "inquire", @@ -1437,7 +1438,6 @@ version = "0.1.0" dependencies = [ "cap-mediafoundation-utils", "inquire", - "parking_lot", "thiserror 1.0.69", "tracing", "windows 0.60.0", @@ -1795,6 +1795,7 @@ name = "cap-frame-converter" version = "0.1.0" dependencies = [ "cap-media-info", + "cidre", "crossbeam-channel", "ffmpeg-next", "flume", diff --git a/crates/camera-directshow/src/lib.rs b/crates/camera-directshow/src/lib.rs index d2166053d68..81043f845de 100644 --- a/crates/camera-directshow/src/lib.rs +++ b/crates/camera-directshow/src/lib.rs @@ -8,7 +8,12 @@ use std::{ ops::Deref, os::windows::ffi::OsStringExt, ptr::{self, null, null_mut}, - sync::OnceLock, + sync::{ + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{Receiver, RecvTimeoutError, SyncSender, TrySendError, sync_channel}, + }, + thread::{self, JoinHandle}, time::Duration, }; use tracing::*; @@ -34,6 +39,10 @@ pub fn initialize_directshow() -> windows_core::Result<()> { unsafe { CoInitialize(None) }.ok() } +fn initialize_directshow_worker() { + let _ = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; +} + pub trait IPinExt { /// # Safety /// Do it correctly @@ -460,7 +469,10 @@ impl VideoInputDevice { .SetFormat(&**format) .map_err(StartCapturingError::Other)?; - let sink_filter = SinkFilter::new(format.clone(), callback); + let dropped = Arc::new(AtomicU64::new(0)); + let (sink_filter, callback_worker) = + SinkFilter::new(format.clone(), callback, dropped.clone()) + .map_err(StartCapturingError::CallbackWorker)?; let input_sink_pin = sink_filter .get_pin(0) @@ -515,6 +527,8 @@ impl VideoInputDevice { graph_builder, output_capture_pin: bound.output_pin, input_sink_pin, + callback_worker, + dropped_samples: dropped, }) } } @@ -525,9 +539,15 @@ pub struct CaptureHandle { graph_builder: IGraphBuilder, output_capture_pin: IPin, input_sink_pin: IPin, + callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + // Chromium: VideoCaptureDeviceWin::StopAndDeallocate pub fn stop_capturing(self) -> windows_core::Result<()> { unsafe { self.media_control.Stop() }?; @@ -537,6 +557,8 @@ impl CaptureHandle { let _ = self.graph_builder.Disconnect(&self.input_sink_pin); } + drop(self.callback_worker); + Ok(()) } } @@ -553,6 +575,8 @@ pub enum StartCapturingError { ConfigureGraph(windows_core::Error), #[error("Run: {0}")] Run(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(std::io::Error), #[error("{0}")] Other(windows_core::Error), } @@ -582,6 +606,11 @@ impl Iterator for VideoMediaTypesIterator<'_> { #[derive(Default, Debug)] pub struct AMMediaType(AM_MEDIA_TYPE); +// The media type owns its format block and is immutable after construction; +// the callback worker only reads it. +unsafe impl Send for AMMediaType {} +unsafe impl Sync for AMMediaType {} + impl AMMediaType { pub fn new(typ: &AM_MEDIA_TYPE) -> Self { Self(unsafe { copy_media_type(typ) }) @@ -593,6 +622,32 @@ impl AMMediaType { std::mem::forget(wrapper); inner } + + fn matches_media_type(&self, other: &AM_MEDIA_TYPE) -> bool { + let a = &self.0; + let b = other; + + if a.majortype != b.majortype + || a.subtype != b.subtype + || a.formattype != b.formattype + || a.cbFormat != b.cbFormat + { + return false; + } + + if a.cbFormat == 0 { + return true; + } + + if a.pbFormat.is_null() || b.pbFormat.is_null() { + return a.pbFormat == b.pbFormat; + } + + unsafe { + std::slice::from_raw_parts(a.pbFormat, a.cbFormat as usize) + == std::slice::from_raw_parts(b.pbFormat, b.cbFormat as usize) + } + } } impl Deref for AMMediaType { @@ -628,13 +683,22 @@ pub struct SinkFilter { } impl SinkFilter { - pub fn new(desired_media_type: AMMediaType, callback: SinkCallback) -> ComObject { + pub(crate) fn new( + desired_media_type: AMMediaType, + callback: SinkCallback, + dropped: Arc, + ) -> std::io::Result<(ComObject, CallbackWorker)> { + let (callback_tx, callback_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let callback_worker = + CallbackWorker::spawn(callback, callback_rx, Arc::new(AtomicBool::new(false)))?; + let this = SinkFilter { state: RefCell::new(State_Stopped), owning_graph: RefCell::new(None), input_pin: SinkInputPin { desired_media_type, - callback: RefCell::new(callback), + callback_tx, + dropped, current_media_type: Default::default(), connected_pin: Default::default(), owner: Default::default(), @@ -646,7 +710,7 @@ impl SinkFilter { // SAFETY: SinkFilter always implements IBaseFilter *this.input_pin.owner.borrow_mut() = Some(this.cast::().unwrap()); - this + Ok((this, callback_worker)) } fn no_of_pins(&self) -> u32 { @@ -814,16 +878,73 @@ pub struct CallbackData<'a> { pub perf_counter: i64, } -pub type SinkCallback = Box; +pub type SinkCallback = Box; + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct OwnedCallbackData { + sample: IMediaSample, + media_type: Arc, + timestamp: Duration, + perf_counter: i64, +} + +unsafe impl Send for OwnedCallbackData {} + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: SinkCallback, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("dshow-camera-callback".to_string()) + .spawn(move || { + initialize_directshow_worker(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(data) => callback(CallbackData { + sample: &data.sample, + media_type: data.media_type.as_ref(), + timestamp: data.timestamp, + perf_counter: data.perf_counter, + }), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} #[implement(IPin, IMemInputPin)] struct SinkInputPin { desired_media_type: AMMediaType, - #[allow(unused)] - current_media_type: RefCell, + current_media_type: RefCell>, connected_pin: RefCell>, owner: RefCell>, - callback: RefCell, + callback_tx: SyncSender, + dropped: Arc, } // impl SinkInputPin { @@ -890,7 +1011,7 @@ impl IPin_Impl for SinkInputPin_Impl { }; *connected_pin = Some(pconnector.clone()); self.current_media_type - .replace(AMMediaType::new(unsafe { &*pmt })); + .replace(Arc::new(AMMediaType::new(unsafe { &*pmt }))); S_OK.ok() } @@ -917,7 +1038,7 @@ impl IPin_Impl for SinkInputPin_Impl { .as_ref() .ok_or(VFW_E_NOT_CONNECTED)?; - unsafe { *pmt = self.current_media_type.borrow().clone().into_inner() }; + unsafe { *pmt = (**self.current_media_type.borrow()).clone().into_inner() }; Ok(()) } @@ -1020,16 +1141,16 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { return Ok(()); }; - unsafe { - if let Ok(new_media_type) = psample.GetMediaType() + let media_type = { + let mut current = self.current_media_type.borrow_mut(); + if let Ok(new_media_type) = unsafe { psample.GetMediaType() } && !new_media_type.is_null() + && !current.matches_media_type(unsafe { &*new_media_type }) { - self.current_media_type - .replace(AMMediaType::new(&*new_media_type)); + *current = Arc::new(AMMediaType::new(unsafe { &*new_media_type })); } - } - - let media_type = self.current_media_type.borrow(); + Arc::clone(&*current) + }; let length = unsafe { psample.GetActualDataLength() }; @@ -1046,12 +1167,19 @@ impl IMemInputPin_Impl for SinkInputPin_Impl { unsafe { psample.GetTime(&mut start_time, &mut end_time) }?; - (self.callback.borrow_mut())(CallbackData { - sample: psample, - media_type: &media_type, + let data = OwnedCallbackData { + sample: psample.clone(), + media_type, timestamp: Duration::from_micros(start_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.callback_tx.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "DirectShow camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-ffmpeg/Cargo.toml b/crates/camera-ffmpeg/Cargo.toml index 6c98a31a488..6e7045baf25 100644 --- a/crates/camera-ffmpeg/Cargo.toml +++ b/crates/camera-ffmpeg/Cargo.toml @@ -16,9 +16,13 @@ cidre = { workspace = true } cap-camera-avfoundation = { path = "../camera-avfoundation" } [target.'cfg(windows)'.dependencies] -windows = { workspace = true } +windows = { workspace = true, features = [ + "Win32_Foundation", + "Win32_Media_MediaFoundation", +] } windows-core = { workspace = true } cap-camera-windows = { path = "../camera-windows" } +cap-mediafoundation-utils = { path = "../mediafoundation-utils" } [dev-dependencies] inquire = "0.7.5" diff --git a/crates/camera-ffmpeg/examples/frame-copy-bench.rs b/crates/camera-ffmpeg/examples/frame-copy-bench.rs new file mode 100644 index 00000000000..ef56f6f12da --- /dev/null +++ b/crates/camera-ffmpeg/examples/frame-copy-bench.rs @@ -0,0 +1,153 @@ +#[cfg(windows)] +fn main() { + use cap_camera::CapturedFrame; + use cap_camera_ffmpeg::{copy_media_buffer_to_ffmpeg, media_buffer_as_ffmpeg}; + use cap_camera_windows::{Frame, PixelFormat}; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use ffmpeg::{format::Pixel, frame::Video}; + use std::time::{Duration, Instant}; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 1000; + + let len = WIDTH * HEIGHT * 3 / 2; + + let surface = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + } + .expect("2D camera buffer"); + { + let mut lock = surface.lock_for_write().expect("surface lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + let surface_frame = Frame::from_media_foundation( + surface, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let captured = CapturedFrame::from_windows_frame(surface_frame, Duration::ZERO); + let native_buffer = captured.native().media_buffer().expect("native buffer"); + + let memory = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("camera buffer"); + { + let mut lock = memory.lock_for_write().expect("buffer lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 31) as u8; + } + } + unsafe { memory.SetCurrentLength(len as u32) }.expect("buffer length"); + let memory_frame = Frame::from_media_foundation( + memory, + PixelFormat::NV12, + WIDTH, + HEIGHT, + false, + Duration::ZERO, + 0, + ); + let copy_captured = CapturedFrame::from_windows_frame(memory_frame, Duration::ZERO); + let copy_bytes = copy_captured.native().bytes().expect("camera bytes"); + + for _ in 0..50 { + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("warmup conversion"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = media_buffer_as_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT, false) + .expect("conversion"); + std::hint::black_box(frame); + } + let zero_copy = start.elapsed(); + + for _ in 0..50 { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("warmup preview copy"); + std::hint::black_box(frame); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let frame = copy_media_buffer_to_ffmpeg(&native_buffer, PixelFormat::NV12, WIDTH, HEIGHT) + .expect("preview copy"); + std::hint::black_box(frame); + } + let preview_copy = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let mut frame = Video::new(Pixel::NV12, WIDTH as u32, HEIGHT as u32); + let stride = frame.stride(0); + for row in 0..HEIGHT { + frame.data_mut(0)[row * stride..row * stride + WIDTH] + .copy_from_slice(©_bytes[row * WIDTH..row * WIDTH + WIDTH]); + } + let uv_stride = frame.stride(1); + let uv_offset = WIDTH * HEIGHT; + for row in 0..HEIGHT / 2 { + frame.data_mut(1)[row * uv_stride..row * uv_stride + WIDTH].copy_from_slice( + ©_bytes[uv_offset + row * WIDTH..uv_offset + row * WIDTH + WIDTH], + ); + } + std::hint::black_box(frame); + } + let copy = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let buffer = captured.native().media_buffer().expect("native buffer"); + std::hint::black_box(buffer); + } + let buffer_clone = start.elapsed(); + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let bytes = captured.native().bytes().expect("native bytes").to_vec(); + std::hint::black_box(bytes); + } + let bytes_copy = start.elapsed(); + + println!( + "zero-copy: {:.3} ms/frame ({:.1} fps)", + zero_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / zero_copy.as_secs_f64() + ); + println!( + "preview copy: {:.3} ms/frame ({:.1} fps)", + preview_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / preview_copy.as_secs_f64() + ); + println!( + "row copy: {:.3} ms/frame ({:.1} fps)", + copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / copy.as_secs_f64() + ); + println!( + "buffer clone: {:.3} ms/frame", + buffer_clone.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); + println!( + "bytes copy: {:.3} ms/frame", + bytes_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64 + ); +} + +#[cfg(not(windows))] +fn main() { + println!("frame-copy-bench requires Windows camera buffers"); +} diff --git a/crates/camera-ffmpeg/src/lib.rs b/crates/camera-ffmpeg/src/lib.rs index 3ec20823a6b..5905200a2b5 100644 --- a/crates/camera-ffmpeg/src/lib.rs +++ b/crates/camera-ffmpeg/src/lib.rs @@ -13,6 +13,76 @@ mod linux; #[cfg(target_os = "linux")] pub use linux::*; +use std::sync::atomic::{AtomicU64, Ordering}; + +#[derive(Debug, Default, Clone, Copy)] +pub struct ConversionStatsSnapshot { + pub zero_copy_frames: u64, + pub copied_frames: u64, + pub copy_nanos: u64, +} + +impl ConversionStatsSnapshot { + pub fn average_copy_micros(&self) -> f64 { + if self.copied_frames == 0 { + 0.0 + } else { + self.copy_nanos as f64 / self.copied_frames as f64 / 1_000.0 + } + } +} + +#[derive(Debug, Default)] +pub struct ConversionStats { + zero_copy_frames: AtomicU64, + copied_frames: AtomicU64, + copy_nanos: AtomicU64, +} + +impl ConversionStats { + pub fn record_zero_copy(&self) { + self.zero_copy_frames.fetch_add(1, Ordering::Relaxed); + self.log_periodically(); + } + + pub fn record_copy(&self, elapsed_nanos: u64) { + self.copied_frames.fetch_add(1, Ordering::Relaxed); + self.copy_nanos.fetch_add(elapsed_nanos, Ordering::Relaxed); + self.log_periodically(); + } + + fn log_periodically(&self) { + let snapshot = self.snapshot(); + let total = snapshot.zero_copy_frames + snapshot.copied_frames; + if total > 0 && total.is_multiple_of(300) { + tracing::debug!( + zero_copy_frames = snapshot.zero_copy_frames, + copied_frames = snapshot.copied_frames, + average_copy_micros = snapshot.average_copy_micros(), + "Camera frame conversion stats" + ); + } + } + + pub fn snapshot(&self) -> ConversionStatsSnapshot { + ConversionStatsSnapshot { + zero_copy_frames: self.zero_copy_frames.load(Ordering::Relaxed), + copied_frames: self.copied_frames.load(Ordering::Relaxed), + copy_nanos: self.copy_nanos.load(Ordering::Relaxed), + } + } +} + +pub static CONVERSION_STATS: ConversionStats = ConversionStats { + zero_copy_frames: AtomicU64::new(0), + copied_frames: AtomicU64::new(0), + copy_nanos: AtomicU64::new(0), +}; + +pub fn conversion_stats() -> ConversionStatsSnapshot { + CONVERSION_STATS.snapshot() +} + pub trait CapturedFrameExt { /// Creates an ffmpeg video frame from the native frame. /// Only size, format, and data are set. diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index 89e42e2a85b..62e303e995a 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -2,7 +2,7 @@ use cap_camera::{CapturedFrame, NativeFrameFormat}; use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; use std::cell::RefCell; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -77,8 +77,9 @@ impl CapturedFrameExt for CapturedFrame { let native = self.native(); let format = native.format; let bytes = native.bytes.as_slice(); + let started = std::time::Instant::now(); - match fourcc_str(format.fourcc).as_str() { + let frame = match fourcc_str(format.fourcc).as_str() { "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), @@ -88,7 +89,10 @@ impl CapturedFrameExt for CapturedFrame { "YV12" => copy_yuv420(bytes, format, true), "MJPG" | "JPEG" => decode_mjpeg(bytes), other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), - } + }?; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } } diff --git a/crates/camera-ffmpeg/src/macos.rs b/crates/camera-ffmpeg/src/macos.rs index a6c11b602bd..eb822a1efc0 100644 --- a/crates/camera-ffmpeg/src/macos.rs +++ b/crates/camera-ffmpeg/src/macos.rs @@ -7,7 +7,7 @@ use std::{ sync::atomic::{AtomicBool, Ordering}, }; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; type CachedScaler = (Pixel, u32, u32, scaling::Context, ffmpeg::frame::Video); @@ -69,7 +69,8 @@ impl CapturedFrameExt for CapturedFrame { pub fn sample_buf_as_ffmpeg( sample_buf: &cm::SampleBuf, ) -> Result { - { + let started = std::time::Instant::now(); + let frame = { let mut image_buf = sample_buf .image_buf() .map(|b| b.retained()) @@ -447,6 +448,9 @@ pub fn sample_buf_as_ffmpeg( } }; - Ok(res) - } + res + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) } diff --git a/crates/camera-ffmpeg/src/windows.rs b/crates/camera-ffmpeg/src/windows.rs index 3b1ee47581c..95ba2cfc48e 100644 --- a/crates/camera-ffmpeg/src/windows.rs +++ b/crates/camera-ffmpeg/src/windows.rs @@ -1,10 +1,16 @@ -use std::{cell::RefCell, collections::VecDeque}; +use std::{cell::RefCell, collections::VecDeque, ffi::c_void, ptr}; use cap_camera::CapturedFrame; use cap_camera_windows::PixelFormat; -use ffmpeg::{Packet, format::Pixel, frame::Video as FFVideo}; +use ffmpeg::{Packet, ffi, format::Pixel, frame::Video as FFVideo}; +use windows::{ + Win32::Media::MediaFoundation::{ + IMF2DBuffer, IMF2DBuffer2, IMFMediaBuffer, MF2DBuffer_LockFlags_Read, + }, + core::Interface, +}; -use crate::CapturedFrameExt; +use crate::{CONVERSION_STATS, CapturedFrameExt}; #[derive(thiserror::Error, Debug)] pub enum AsFFmpegError { @@ -229,6 +235,377 @@ fn src_y(y: usize, height: usize, is_bottom_up: bool) -> usize { if is_bottom_up { height - y - 1 } else { y } } +#[derive(Clone, Copy)] +struct MfPlane { + ptr: *mut u8, + len: usize, + stride: usize, +} + +struct MfPlanes { + planes: [MfPlane; 3], + len: usize, +} + +impl MfPlanes { + fn new() -> Self { + Self { + planes: [MfPlane { + ptr: ptr::null_mut(), + len: 0, + stride: 0, + }; 3], + len: 0, + } + } + + fn push(&mut self, plane: MfPlane) { + if self.len < self.planes.len() { + self.planes[self.len] = plane; + self.len += 1; + } + } + + fn as_slice(&self) -> &[MfPlane] { + &self.planes[..self.len] + } +} + +enum MfLock { + TwoD2(IMF2DBuffer2), + TwoD(IMF2DBuffer), + Contiguous, +} + +struct WrappedMfBuffer { + _buffer: IMFMediaBuffer, + lock: MfLock, +} + +unsafe impl Send for WrappedMfBuffer {} +unsafe impl Sync for WrappedMfBuffer {} + +impl Drop for WrappedMfBuffer { + fn drop(&mut self) { + unsafe { + match &self.lock { + MfLock::TwoD2(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::TwoD(buffer) => { + let _ = buffer.Unlock2D(); + } + MfLock::Contiguous => { + let _ = self._buffer.Unlock(); + } + } + } + } +} + +unsafe extern "C" fn release_wrapped_mf_buffer(opaque: *mut c_void, _data: *mut u8) { + if !opaque.is_null() { + drop(unsafe { Box::from_raw(opaque as *mut WrappedMfBuffer) }); + } +} + +fn mf_pixel_format(format: PixelFormat) -> Option { + Some(match format { + PixelFormat::NV12 => Pixel::NV12, + PixelFormat::NV21 => Pixel::NV21, + PixelFormat::YUYV422 => Pixel::YUYV422, + PixelFormat::UYVY422 => Pixel::UYVY422, + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => { + return None; + } + PixelFormat::ARGB | PixelFormat::RGB32 => Pixel::BGRA, + PixelFormat::RGB24 | PixelFormat::BGR24 => Pixel::BGR24, + PixelFormat::GRAY8 => Pixel::GRAY8, + PixelFormat::GRAY16 => Pixel::GRAY16LE, + PixelFormat::RGB565 => Pixel::RGB565LE, + PixelFormat::P010 => Pixel::P010LE, + }) +} + +fn mf_row_bytes(format: PixelFormat, width: usize) -> Option { + Some(match format { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::YUV420P | PixelFormat::GRAY8 => width, + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 + | PixelFormat::P010 => width.checked_mul(2)?, + PixelFormat::RGB24 | PixelFormat::BGR24 => width.checked_mul(3)?, + PixelFormat::ARGB | PixelFormat::RGB32 => width.checked_mul(4)?, + PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => return None, + }) +} + +fn mf_planes( + format: PixelFormat, + width: usize, + height: usize, + data: *mut u8, + len: usize, + stride: usize, +) -> Option { + let row_bytes = mf_row_bytes(format, width)?; + if stride < row_bytes || height == 0 { + return None; + } + + let plane = |ptr: *mut u8, len: usize| MfPlane { ptr, len, stride }; + let y_len = stride.checked_mul(height)?; + let mut planes = MfPlanes::new(); + + match format { + PixelFormat::NV12 | PixelFormat::NV21 | PixelFormat::P010 => { + let uv_len = stride.checked_mul(height / 2)?; + if len < y_len.checked_add(uv_len)? { + return None; + } + planes.push(plane(data, y_len)); + planes.push(plane(unsafe { data.add(y_len) }, uv_len)); + Some(planes) + } + PixelFormat::YUYV422 + | PixelFormat::UYVY422 + | PixelFormat::ARGB + | PixelFormat::RGB32 + | PixelFormat::RGB24 + | PixelFormat::BGR24 + | PixelFormat::GRAY8 + | PixelFormat::GRAY16 + | PixelFormat::RGB565 => { + if len < y_len { + return None; + } + planes.push(plane(data, y_len)); + Some(planes) + } + PixelFormat::YUV420P | PixelFormat::YV12 | PixelFormat::MJPEG | PixelFormat::H264 => None, + } +} + +fn lock_mf_buffer( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option<(WrappedMfBuffer, MfPlanes)> { + unsafe { + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + let mut start = ptr::null_mut(); + let mut len = 0u32; + if buffer_2d + .Lock2DSize( + MF2DBuffer_LockFlags_Read, + &mut scanline, + &mut pitch, + &mut start, + &mut len, + ) + .is_ok() + { + if !start.is_null() + && pitch > 0 + && len > 0 + && let Some(planes) = + mf_planes(format, width, height, start, len as usize, pitch as usize) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD2(buffer_2d), + }, + planes, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + if let Ok(buffer_2d) = buffer.cast::() { + let mut scanline = ptr::null_mut(); + let mut pitch = 0i32; + if buffer_2d.Lock2D(&mut scanline, &mut pitch).is_ok() { + if !scanline.is_null() + && pitch > 0 + && let Ok(len) = buffer_2d.GetContiguousLength() + && let Some(planes) = mf_planes( + format, + width, + height, + scanline, + len as usize, + pitch as usize, + ) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::TwoD(buffer_2d), + }, + planes, + )); + } + + let _ = buffer_2d.Unlock2D(); + } + } + + let mut data = ptr::null_mut(); + let mut len = 0u32; + if buffer.Lock(&mut data, None, Some(&mut len)).is_ok() { + if !data.is_null() + && len > 0 + && let Some(row_bytes) = mf_row_bytes(format, width) + && let Some(planes) = + mf_planes(format, width, height, data, len as usize, row_bytes) + { + return Some(( + WrappedMfBuffer { + _buffer: buffer.clone(), + lock: MfLock::Contiguous, + }, + planes, + )); + } + + let _ = buffer.Unlock(); + } + } + + None +} + +pub fn media_buffer_as_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, +) -> Option { + if !should_use_2d_lock(format, is_bottom_up) || width == 0 || height == 0 { + return None; + } + + let pixel = mf_pixel_format(format)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + wrap_mf_frame(&planes, pixel, width as u32, height as u32, owner) +} + +// UYVY and NV21 use the format-converting copy paths, which lock with +// IMFMediaBuffer::Lock; all consumers of a buffer must use the same lock type. +pub fn should_use_2d_lock(format: PixelFormat, is_bottom_up: bool) -> bool { + !is_bottom_up + && format != PixelFormat::UYVY422 + && format != PixelFormat::NV21 + && mf_pixel_format(format).is_some() +} + +pub fn with_media_buffer_data( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, + f: impl FnOnce(&[u8], usize, usize) -> R, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let data = unsafe { std::slice::from_raw_parts(planes[0].ptr, total_len) }; + let primary_stride = planes[0].stride; + let secondary_stride = planes.get(1).map_or(primary_stride, |plane| plane.stride); + let result = f(data, primary_stride, secondary_stride); + drop(owner); + Some(result) +} + +pub fn copy_media_buffer_to_ffmpeg( + buffer: &IMFMediaBuffer, + format: PixelFormat, + width: usize, + height: usize, +) -> Option { + if !should_use_2d_lock(format, false) { + return None; + } + + let pixel = mf_pixel_format(format)?; + let row_bytes = mf_row_bytes(format, width)?; + let (owner, planes) = lock_mf_buffer(buffer, format, width, height)?; + let planes = planes.as_slice(); + let mut frame = FFVideo::new(pixel, width as u32, height as u32); + + for (index, plane) in planes.iter().enumerate() { + let plane_height = plane.len / plane.stride; + let destination_stride = frame.stride(index); + let destination = frame.data_mut(index); + + for row in 0..plane_height { + let source = + unsafe { std::slice::from_raw_parts(plane.ptr.add(row * plane.stride), row_bytes) }; + let destination_start = row * destination_stride; + destination[destination_start..destination_start + row_bytes].copy_from_slice(source); + } + } + + drop(owner); + Some(frame) +} + +fn wrap_mf_frame( + planes: &MfPlanes, + pixel: Pixel, + width: u32, + height: u32, + owner: WrappedMfBuffer, +) -> Option { + let planes = planes.as_slice(); + let total_len = planes.iter().map(|plane| plane.len).sum::(); + let opaque = Box::into_raw(Box::new(owner)) as *mut c_void; + let buffer = unsafe { + ffi::av_buffer_create( + planes[0].ptr, + total_len, + Some(release_wrapped_mf_buffer), + opaque, + ffi::AV_BUFFER_FLAG_READONLY, + ) + }; + + if buffer.is_null() { + unsafe { + drop(Box::from_raw(opaque as *mut WrappedMfBuffer)); + } + return None; + } + + let mut frame = FFVideo::empty(); + let ptr = unsafe { frame.as_mut_ptr() }; + let format: ffi::AVPixelFormat = pixel.into(); + unsafe { + (*ptr).format = format as i32; + (*ptr).width = width as i32; + (*ptr).height = height as i32; + (*ptr).buf[0] = buffer; + for (index, plane) in planes.iter().enumerate() { + (*ptr).data[index] = plane.ptr; + (*ptr).linesize[index] = plane.stride as i32; + } + } + + Some(frame) +} + impl CapturedFrameExt for CapturedFrame { fn as_ffmpeg(&self) -> Result { let native = self.native(); @@ -240,13 +617,24 @@ impl CapturedFrameExt for CapturedFrame { return Err(AsFFmpegError::Empty); } + let started = std::time::Instant::now(); + + if should_use_2d_lock(native.pixel_format, is_bottom_up) + && let Some(buffer) = native.media_buffer() + && let Some(frame) = + copy_media_buffer_to_ffmpeg(&buffer, native.pixel_format, width, height) + { + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + return Ok(frame); + } + let bytes = native.bytes().map_err(AsFFmpegError::FailedToGetBytes)?; if bytes.is_empty() { return Err(AsFFmpegError::Empty); } - Ok(match native.pixel_format { + let frame = match native.pixel_format { PixelFormat::YUV420P => { let mut ff_frame = FFVideo::new(Pixel::YUV420P, width as u32, height as u32); @@ -502,6 +890,171 @@ impl CapturedFrameExt for CapturedFrame { ff_frame } PixelFormat::H264 => decode_h264(&bytes)?, - }) + }; + + CONVERSION_STATS.record_copy(started.elapsed().as_nanos() as u64); + Ok(frame) + } +} + +#[cfg(test)] +mod conversion_tests { + use super::*; + use cap_camera_windows::Frame; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use windows::Win32::Media::MediaFoundation::{MFCreate2DMediaBuffer, MFCreateMemoryBuffer}; + + #[test] + fn nv12_2d_buffer_can_be_wrapped_twice() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let first = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("first frame"); + let second = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + + assert!(second.is_some(), "second lock failed"); + drop(first); + let third = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ); + assert!(third.is_some(), "third lock failed"); + } + + #[test] + fn copy_conversion_can_nest_inside_zero_copy_frame() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let buffer = + unsafe { MFCreate2DMediaBuffer(width, height, u32::from_le_bytes(*b"NV12"), false) } + .expect("2D buffer"); + + let zero_copy = media_buffer_as_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + false, + ) + .expect("zero-copy frame"); + let copied = copy_media_buffer_to_ffmpeg( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + ) + .expect("copied frame"); + + assert_eq!(copied.format(), Pixel::NV12); + assert_eq!(copied.width(), width); + assert_eq!(copied.height(), height); + + let data_len = with_media_buffer_data( + &buffer, + PixelFormat::NV12, + width as usize, + height as usize, + |data, primary_stride, secondary_stride| { + assert_eq!(primary_stride, secondary_stride); + assert!(primary_stride >= width as usize); + data.len() + }, + ) + .expect("locked data"); + + assert!(data_len >= width as usize * height as usize * 3 / 2); + drop(zero_copy); + } + + #[test] + fn nv12_media_buffer_is_wrapped_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for row in 0..height { + for column in 0..width { + lock[row * stride + column] = (row + column) as u8; + } + } + let uv = stride * height; + for row in 0..height / 2 { + for column in 0..width { + lock[uv + row * stride + column] = (row * 3 + column) as u8; + } + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = media_buffer_as_ffmpeg(&buffer, PixelFormat::NV12, width, height, false) + .expect("zero-copy frame"); + + assert_eq!(frame.format(), Pixel::NV12); + assert_eq!(frame.width(), width as u32); + assert_eq!(frame.height(), height as u32); + assert_eq!(frame.stride(0), stride); + assert_eq!(frame.stride(1), stride); + assert_eq!(frame.data(0)[0], 0); + assert_eq!(frame.data(0)[stride + 1], 2); + assert_eq!(frame.data(1)[0], 0); + assert_eq!(frame.data(1)[stride + 2], 5); + } + + #[test] + fn as_ffmpeg_keeps_nv12_for_nv21_frames() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let frame = Frame::from_media_foundation( + buffer, + PixelFormat::NV21, + width, + height, + false, + std::time::Duration::ZERO, + 0, + ); + let captured = + cap_camera::CapturedFrame::from_windows_frame(frame, std::time::Duration::ZERO); + let converted = captured.as_ffmpeg().expect("conversion"); + + assert_eq!(converted.format(), Pixel::NV12); + assert_eq!(converted.width(), width as u32); + assert_eq!(converted.height(), height as u32); } } diff --git a/crates/camera-mediafoundation/Cargo.toml b/crates/camera-mediafoundation/Cargo.toml index 36ee6fdd9d0..0bee6fe27df 100644 --- a/crates/camera-mediafoundation/Cargo.toml +++ b/crates/camera-mediafoundation/Cargo.toml @@ -7,7 +7,6 @@ license = "MIT" [dependencies] tracing.workspace = true thiserror.workspace = true -parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(windows)'.dependencies] diff --git a/crates/camera-mediafoundation/src/lib.rs b/crates/camera-mediafoundation/src/lib.rs index 246b1c311e9..4378ef24b5c 100644 --- a/crates/camera-mediafoundation/src/lib.rs +++ b/crates/camera-mediafoundation/src/lib.rs @@ -2,7 +2,6 @@ #![allow(non_snake_case)] use cap_mediafoundation_utils::*; -use parking_lot::Mutex; use std::{ ffi::OsString, fmt::Display, @@ -11,9 +10,13 @@ use std::{ os::windows::ffi::OsStringExt, slice::from_raw_parts, sync::{ - OnceLock, - mpsc::{Receiver, Sender, channel}, + Arc, OnceLock, + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc::{ + Receiver, RecvTimeoutError, Sender, SyncSender, TrySendError, channel, sync_channel, + }, }, + thread::{self, JoinHandle}, time::Duration, }; use windows::Win32::{ @@ -144,6 +147,8 @@ pub enum StartCapturingError { ConfigureSink(windows_core::Error), #[error("StartPreview: {0}")] StartPreview(windows_core::Error), + #[error("CallbackWorker: {0}")] + CallbackWorker(#[from] std::io::Error), } impl Device { @@ -233,7 +238,7 @@ impl Device { pub fn start_capturing( &self, requested_format: &IMFMediaType, - callback: Box, + callback: Box, ) -> Result { let media_source = self .media_source() @@ -252,9 +257,14 @@ impl Device { .map_err(StartCapturingError::CreateEngine)?; let (event_tx, event_rx) = channel(); + let (sample_tx, sample_rx) = sync_channel(CALLBACK_QUEUE_DEPTH); + let dropped = Arc::new(AtomicU64::new(0)); + let callback_worker = + CallbackWorker::spawn(callback, sample_rx, Arc::new(AtomicBool::new(false)))?; let video_callback = VideoCallback { event_tx, - sample_callback: Mutex::new(callback), + samples: sample_tx, + dropped: dropped.clone(), } .into_object(); @@ -375,7 +385,12 @@ impl Device { wait_for_event(&event_rx, CaptureEngineEventVariant::PreviewStarted) .map_err(|v| StartCapturingError::StartPreview(v.into()))?; - Ok(CaptureHandle { engine, event_rx }) + Ok(CaptureHandle { + engine: Some(engine), + event_rx, + callback_worker, + dropped_samples: dropped, + }) } } } @@ -408,8 +423,10 @@ fn retry_on_invalid_request( } pub struct CaptureHandle { + engine: Option, event_rx: Receiver, - engine: IMFCaptureEngine, + callback_worker: CallbackWorker, + dropped_samples: Arc, } impl CaptureHandle { @@ -417,8 +434,18 @@ impl CaptureHandle { &self.event_rx } - pub fn stop_capturing(self) -> windows_core::Result<()> { - unsafe { self.engine.StopPreview() } + pub fn dropped_samples(&self) -> u64 { + self.dropped_samples.load(Ordering::Relaxed) + } + + pub fn stop_capturing(mut self) -> windows_core::Result<()> { + let result = if let Some(engine) = self.engine.take() { + unsafe { engine.StopPreview() } + } else { + Ok(()) + }; + drop(self.callback_worker); + result } } @@ -570,10 +597,56 @@ pub struct CallbackData { pub perf_counter: i64, } +unsafe impl Send for CallbackData {} + +const CALLBACK_QUEUE_DEPTH: usize = 4; + +struct CallbackWorker { + stop: Arc, + handle: Option>, +} + +impl CallbackWorker { + fn spawn( + mut callback: Box, + samples: Receiver, + stop: Arc, + ) -> std::io::Result { + let worker_stop = stop.clone(); + let handle = thread::Builder::new() + .name("mf-camera-callback".to_string()) + .spawn(move || { + cap_mediafoundation_utils::thread_init(); + while !worker_stop.load(Ordering::Acquire) { + match samples.recv_timeout(Duration::from_millis(100)) { + Ok(sample) => callback(sample), + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + })?; + + Ok(Self { + stop, + handle: Some(handle), + }) + } +} + +impl Drop for CallbackWorker { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + #[implement(IMFCaptureEngineOnSampleCallback, IMFCaptureEngineOnEventCallback)] struct VideoCallback { event_tx: Sender, - sample_callback: Mutex>, + samples: SyncSender, + dropped: Arc, } impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { @@ -585,15 +658,20 @@ impl IMFCaptureEngineOnSampleCallback_Impl for VideoCallback_Impl { return Ok(()); }; - let mut callback = self.sample_callback.lock(); - let sample_time = unsafe { sample.GetSampleTime() }?; - (callback)(CallbackData { + let data = CallbackData { sample: sample.clone(), timestamp: Duration::from_micros(sample_time as u64 / 10), perf_counter, - }); + }; + + if let Err(TrySendError::Full(_)) = self.samples.try_send(data) { + let dropped = self.dropped.fetch_add(1, Ordering::Relaxed) + 1; + if dropped == 1 || dropped.is_multiple_of(100) { + tracing::warn!(dropped, "Media Foundation camera callback queue full"); + } + } Ok(()) } diff --git a/crates/camera-windows/src/lib.rs b/crates/camera-windows/src/lib.rs index b1004ae2d6c..c1badbd9e68 100644 --- a/crates/camera-windows/src/lib.rs +++ b/crates/camera-windows/src/lib.rs @@ -173,6 +173,13 @@ pub enum CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + match self { + CaptureHandle::MediaFoundation(handle) => handle.dropped_samples(), + CaptureHandle::DirectShow(handle) => handle.dropped_samples(), + } + } + pub fn stop_capturing(self) -> windows_core::Result<()> { match self { CaptureHandle::MediaFoundation(handle) => handle.stop_capturing(), @@ -236,8 +243,10 @@ impl VideoDeviceInfo { .max_by_key(|(w, h)| (*w as u64) * (*h as u64)) } - pub fn find_best_format(&self, preference: &FormatPreference) -> Option { - let formats = self.formats(); + fn best_format_in( + formats: &[VideoFormat], + preference: &FormatPreference, + ) -> Option { if formats.is_empty() { return None; } @@ -267,11 +276,19 @@ impl VideoDeviceInfo { format_priority + resolution_score + fps_score }; - formats.into_iter().max_by_key(score_format) + formats + .iter() + .max_by_key(|format| score_format(format)) + .cloned() + } + + pub fn find_best_format(&self, preference: &FormatPreference) -> Option { + Self::best_format_in(&self.formats(), preference) } pub fn find_format_with_fallback(&self, preference: &FormatPreference) -> Option { - if let Some(format) = self.find_best_format(preference) { + let formats = self.formats(); + if let Some(format) = Self::best_format_in(&formats, preference) { return Some(format); } @@ -284,7 +301,6 @@ impl VideoDeviceInfo { PixelFormat::YUV420P, ]; - let formats = self.formats(); for fallback_pixel_format in fallback_formats { if let Some(format) = formats .iter() @@ -319,7 +335,7 @@ impl VideoDeviceInfo { pub fn start_capturing( self, format: &VideoFormatInner, - mut callback: impl FnMut(Frame) + 'static, + mut callback: impl FnMut(Frame) + Send + 'static, ) -> Result { let res = match (self.inner, &format) { ( @@ -327,6 +343,10 @@ impl VideoDeviceInfo { VideoFormatInner::MediaFoundation(mf_format), ) => { let format = VideoFormat::new_mf(mf_format.clone())?; + let width = format.width() as usize; + let height = format.height() as usize; + let is_bottom_up = format.is_bottom_up; + let pixel_format = format.pixel_format; let handle = device.start_capturing( mf_format, @@ -340,10 +360,10 @@ impl VideoDeviceInfo { callback(Frame { inner: FrameInner::MediaFoundation(buffer), - width: format.width() as usize, - height: format.height() as usize, - is_bottom_up: format.is_bottom_up, - pixel_format: format.pixel_format, + width, + height, + is_bottom_up, + pixel_format, timestamp: data.timestamp, perf_counter: data.perf_counter, }) @@ -476,6 +496,41 @@ impl Frame { FrameInner::MediaFoundation(buffer) => Ok(FrameBytes::MediaFoundation(buffer.lock()?)), } } + + pub fn media_buffer(&self) -> Option { + match &self.inner { + FrameInner::MediaFoundation(buffer) => Some(buffer.clone()), + FrameInner::DirectShow(_) => None, + } + } + + pub fn directshow_sample(&self) -> Option { + match &self.inner { + FrameInner::DirectShow(sample) => Some(sample.clone()), + FrameInner::MediaFoundation(_) => None, + } + } + + #[doc(hidden)] + pub fn from_media_foundation( + buffer: IMFMediaBuffer, + pixel_format: PixelFormat, + width: usize, + height: usize, + is_bottom_up: bool, + timestamp: Duration, + perf_counter: i64, + ) -> Self { + Self { + pixel_format, + width, + height, + is_bottom_up, + timestamp, + perf_counter, + inner: FrameInner::MediaFoundation(buffer), + } + } } pub enum FrameBytes<'a> { diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index 98b2875060b..e4e9591c032 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -230,6 +230,15 @@ impl CapturedFrame { pub fn native(&self) -> &NativeCapturedFrame { &self.native } + + #[cfg(windows)] + #[doc(hidden)] + pub fn from_windows_frame(frame: cap_camera_windows::Frame, timestamp: Duration) -> Self { + Self { + native: NativeCapturedFrame::from_frame(frame), + timestamp, + } + } } impl CameraInfo { @@ -274,7 +283,28 @@ pub struct CaptureHandle { } impl CaptureHandle { + pub fn dropped_samples(&self) -> u64 { + #[cfg(windows)] + { + self.native + .as_ref() + .map(NativeCaptureHandle::dropped_samples) + .unwrap_or(0) + } + #[cfg(not(windows))] + { + 0 + } + } + pub fn stop_capturing(mut self) -> Result<(), String> { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture stopped with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing()?; } @@ -284,6 +314,13 @@ impl CaptureHandle { impl Drop for CaptureHandle { fn drop(&mut self) { + let dropped_samples = self.dropped_samples(); + if dropped_samples > 0 { + tracing::debug!( + dropped_samples, + "Camera capture released with dropped callback samples" + ); + } if let Some(feed) = self.native.take() { feed.stop_capturing().ok(); } diff --git a/crates/camera/src/windows.rs b/crates/camera/src/windows.rs index a99d3d7b81f..73c9264cec2 100644 --- a/crates/camera/src/windows.rs +++ b/crates/camera/src/windows.rs @@ -64,12 +64,18 @@ impl ModelID { #[derive(Debug)] pub struct NativeCapturedFrame(cap_camera_windows::Frame); +impl NativeCapturedFrame { + pub(crate) fn from_frame(frame: cap_camera_windows::Frame) -> Self { + Self(frame) + } +} + pub type NativeCaptureHandle = WindowsCaptureHandle; pub(super) fn start_capturing_impl( camera: &CameraInfo, format: Format, - mut callback: impl FnMut(CapturedFrame) + 'static, + mut callback: impl FnMut(CapturedFrame) + Send + 'static, ) -> Result { let device = find_device(camera)?.ok_or(StartCapturingError::DeviceNotFound)?; @@ -90,6 +96,10 @@ pub struct WindowsCaptureHandle { } impl WindowsCaptureHandle { + pub fn dropped_samples(&self) -> u64 { + self.inner.dropped_samples() + } + pub fn stop_capturing(self) -> Result<(), String> { self.inner.stop_capturing().map_err(|e| e.to_string()) } diff --git a/crates/enc-mediafoundation/src/video/h264.rs b/crates/enc-mediafoundation/src/video/h264.rs index 9d34e047605..b7b69e3c757 100644 --- a/crates/enc-mediafoundation/src/video/h264.rs +++ b/crates/enc-mediafoundation/src/video/h264.rs @@ -240,6 +240,10 @@ impl From for EncoderRuntimeError { unsafe impl Send for H264Encoder {} impl H264Encoder { + pub fn input_texture(&self) -> &ID3D11Texture2D { + self.video_processor.input_texture() + } + #[allow(clippy::too_many_arguments)] fn new_with_scaled_output_with_flags( d3d_device: &ID3D11Device, @@ -710,7 +714,12 @@ impl H264Encoder { } let process_result = (|| -> windows::core::Result<()> { - self.video_processor.process_texture(&texture)?; + if texture.as_raw() != self.video_processor.input_texture().as_raw() + { + self.video_processor.process_texture(&texture)?; + } else { + self.video_processor.process_input_texture()?; + } let input_buffer = MFCreateDXGISurfaceBuffer( &ID3D11Texture2D::IID, self.video_processor.output_texture(), diff --git a/crates/enc-mediafoundation/src/video/video_processor.rs b/crates/enc-mediafoundation/src/video/video_processor.rs index fe26d73f337..147532a3796 100644 --- a/crates/enc-mediafoundation/src/video/video_processor.rs +++ b/crates/enc-mediafoundation/src/video/video_processor.rs @@ -218,6 +218,10 @@ impl VideoProcessor { &self.video_output_texture } + pub fn input_texture(&self) -> &ID3D11Texture2D { + &self.video_input_texture + } + pub fn process_texture( &mut self, input_texture: &ID3D11Texture2D, @@ -226,18 +230,23 @@ impl VideoProcessor { // texture that matches the input size we were initialized with. unsafe { - // Copy the texture to the video input texture self.d3d_context .CopyResource(&self.video_input_texture, input_texture); + } - // Convert to NV12 - let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { - Enable: true.into(), - OutputIndex: 0, - InputFrameOrField: 0, - pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), - ..Default::default() - }; + self.process_input_texture() + } + + pub fn process_input_texture(&mut self) -> windows::core::Result<()> { + let video_stream = D3D11_VIDEO_PROCESSOR_STREAM { + Enable: true.into(), + OutputIndex: 0, + InputFrameOrField: 0, + pInputSurface: ManuallyDrop::new(Some(self.video_input.clone())), + ..Default::default() + }; + + unsafe { self.video_context.VideoProcessorBlt( &self.video_processor, &self.video_output, diff --git a/crates/frame-converter/Cargo.toml b/crates/frame-converter/Cargo.toml index 3d363ef7098..be104b4593f 100644 --- a/crates/frame-converter/Cargo.toml +++ b/crates/frame-converter/Cargo.toml @@ -19,6 +19,7 @@ parking_lot = "0.12" workspace-hack = { version = "0.1", path = "../workspace-hack" } [target.'cfg(target_os = "macos")'.dependencies] +cidre = { workspace = true } [target.'cfg(target_os = "windows")'.dependencies] windows = { workspace = true, features = [ diff --git a/crates/frame-converter/src/videotoolbox.rs b/crates/frame-converter/src/videotoolbox.rs index 8d4d8e28d87..b16a3b65f42 100644 --- a/crates/frame-converter/src/videotoolbox.rs +++ b/crates/frame-converter/src/videotoolbox.rs @@ -1,4 +1,5 @@ use crate::{ConversionConfig, ConvertError, ConverterBackend, FrameConverter}; +use cidre::{arc, cf, cv, os, vt}; use ffmpeg::{format::Pixel, frame}; use parking_lot::Mutex; use std::{ @@ -7,100 +8,43 @@ use std::{ sync::atomic::{AtomicBool, AtomicU64, Ordering}, }; -type CFAllocatorRef = *const c_void; -type CFDictionaryRef = *const c_void; type CVPixelBufferRef = *mut c_void; -type VTPixelTransferSessionRef = *mut c_void; -type OSStatus = i32; - -const K_CV_RETURN_SUCCESS: i32 = 0; - -const K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8: u32 = 0x79757679; -const K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE: u32 = 0x34323076; -const K_CV_PIXEL_FORMAT_TYPE_2VUY: u32 = 0x32767579; -const K_CV_PIXEL_FORMAT_TYPE_32_BGRA: u32 = 0x42475241; -const K_CV_PIXEL_FORMAT_TYPE_32_ARGB: u32 = 0x00000020; - -#[link(name = "CoreFoundation", kind = "framework")] -unsafe extern "C" { - fn CFRelease(cf: *const c_void); -} #[link(name = "CoreVideo", kind = "framework")] unsafe extern "C" { - fn CVPixelBufferCreate( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferCreateWithBytes( - allocator: CFAllocatorRef, - width: usize, - height: usize, - pixel_format_type: u32, - base_address: *mut c_void, - bytes_per_row: usize, - release_callback: *const c_void, - release_ref_con: *const c_void, - pixel_buffer_attributes: CFDictionaryRef, - pixel_buffer_out: *mut CVPixelBufferRef, - ) -> i32; - - fn CVPixelBufferRelease(pixel_buffer: CVPixelBufferRef); - - fn CVPixelBufferLockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - fn CVPixelBufferUnlockBaseAddress(pixel_buffer: CVPixelBufferRef, lock_flags: u64) -> i32; - - fn CVPixelBufferGetBaseAddressOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> *mut u8; - fn CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetHeightOfPlane(pixel_buffer: CVPixelBufferRef, plane: usize) -> usize; - fn CVPixelBufferGetPlaneCount(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetBaseAddress(pixel_buffer: CVPixelBufferRef) -> *mut u8; - fn CVPixelBufferGetBytesPerRow(pixel_buffer: CVPixelBufferRef) -> usize; - fn CVPixelBufferGetHeight(pixel_buffer: CVPixelBufferRef) -> usize; -} - -#[link(name = "VideoToolbox", kind = "framework")] -unsafe extern "C" { - fn VTPixelTransferSessionCreate( - allocator: CFAllocatorRef, - pixel_transfer_session_out: *mut VTPixelTransferSessionRef, - ) -> OSStatus; - - fn VTPixelTransferSessionInvalidate(session: VTPixelTransferSessionRef); - - fn VTPixelTransferSessionTransferImage( - session: VTPixelTransferSessionRef, - source_buffer: CVPixelBufferRef, - destination_buffer: CVPixelBufferRef, - ) -> OSStatus; + fn CVPixelBufferGetBaseAddress(pixel_buffer: &cv::PixelBuf) -> *mut u8; + fn CVPixelBufferGetBytesPerRow(pixel_buffer: &cv::PixelBuf) -> usize; + fn CVPixelBufferGetHeight(pixel_buffer: &cv::PixelBuf) -> usize; } -fn pixel_to_cv_format(pixel: Pixel) -> Option { - match pixel { - Pixel::YUYV422 => Some(K_CV_PIXEL_FORMAT_TYPE_422_YP_CB_YP_CR8), - Pixel::UYVY422 => Some(K_CV_PIXEL_FORMAT_TYPE_2VUY), - Pixel::NV12 => Some(K_CV_PIXEL_FORMAT_TYPE_420_YP_CB_CR8_BI_PLANAR_VIDEO_RANGE), - Pixel::BGRA => Some(K_CV_PIXEL_FORMAT_TYPE_32_BGRA), - Pixel::ARGB => Some(K_CV_PIXEL_FORMAT_TYPE_32_ARGB), - _ => None, - } +fn pixel_to_cv_format(pixel: Pixel) -> Option { + Some(match pixel { + Pixel::YUYV422 => cv::PixelFormat(os::Type::from_be_bytes(*b"yuvs")), + Pixel::UYVY422 => cv::PixelFormat::_2VUY, + Pixel::NV12 => cv::PixelFormat::_420V, + Pixel::BGRA => cv::PixelFormat::_32_BGRA, + Pixel::ARGB => cv::PixelFormat::_32_ARGB, + _ => return None, + }) } -struct SessionHandle(VTPixelTransferSessionRef); +struct SessionHandle(arc::R); unsafe impl Send for SessionHandle {} +unsafe impl Sync for SessionHandle {} + +struct PoolHandle(arc::R); + +unsafe impl Send for PoolHandle {} +unsafe impl Sync for PoolHandle {} pub struct VideoToolboxConverter { session: Mutex, + pool: Option, input_format: Pixel, - input_cv_format: u32, + input_cv_format: cv::PixelFormat, output_format: Pixel, - output_cv_format: u32, + output_cv_format: cv::PixelFormat, input_width: u32, input_height: u32, output_width: u32, @@ -109,6 +53,36 @@ pub struct VideoToolboxConverter { verified_hardware: AtomicBool, } +extern "C" fn noop_release(_: *mut c_void, _: *const *const c_void) {} + +fn create_pool( + width: u32, + height: u32, + format: cv::PixelFormat, +) -> Result { + let width = cf::Number::from_usize(width as usize); + let height = cf::Number::from_usize(height as usize); + let io_surface = cf::Dictionary::new(); + let attrs = cf::DictionaryOf::::with_keys_values( + &[ + cv::pixel_buffer::keys::pixel_format(), + cv::pixel_buffer::keys::width(), + cv::pixel_buffer::keys::height(), + cv::pixel_buffer::keys::io_surf_props(), + ], + &[ + format.as_ref(), + width.as_type_ref(), + height.as_type_ref(), + io_surface.as_type_ref(), + ], + ); + + cv::PixelBufPool::new(None, Some(&**attrs)) + .map(PoolHandle) + .map_err(|error| ConvertError::HardwareUnavailable(format!("CVPixelBufferPool: {error}"))) +} + impl VideoToolboxConverter { pub fn new(config: ConversionConfig) -> Result { let input_cv_format = pixel_to_cv_format(config.input_format).ok_or( @@ -119,20 +93,18 @@ impl VideoToolboxConverter { ConvertError::UnsupportedFormat(config.input_format, config.output_format), )?; - let mut session: VTPixelTransferSessionRef = ptr::null_mut(); - let status = unsafe { VTPixelTransferSessionCreate(ptr::null(), &mut session) }; - - if status != 0 { - return Err(ConvertError::HardwareUnavailable(format!( - "VTPixelTransferSessionCreate failed with status: {status}" - ))); - } + let mut session = vt::PixelTransferSession::new().map_err(|error| { + ConvertError::HardwareUnavailable(format!("VTPixelTransferSessionCreate: {error}")) + })?; + let _ = session.set_realtime(true); - if session.is_null() { - return Err(ConvertError::HardwareUnavailable( - "VTPixelTransferSessionCreate returned null session".to_string(), - )); - } + let pool = match create_pool(config.output_width, config.output_height, output_cv_format) { + Ok(pool) => Some(pool), + Err(error) => { + tracing::debug!("VideoToolbox pixel buffer pool unavailable: {error}"); + None + } + }; tracing::debug!( "VideoToolbox converter initialized: {:?} {}x{} -> {:?} {}x{}", @@ -146,6 +118,7 @@ impl VideoToolboxConverter { Ok(Self { session: Mutex::new(SessionHandle(session)), + pool, input_format: config.input_format, input_cv_format, output_format: config.output_format, @@ -162,119 +135,100 @@ impl VideoToolboxConverter { fn create_input_pixel_buffer( &self, input: &frame::Video, - ) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - + ) -> Result, ConvertError> { let base_address = input.data(0).as_ptr() as *mut c_void; let bytes_per_row = input.stride(0); - let status = unsafe { - CVPixelBufferCreateWithBytes( - ptr::null(), - self.input_width as usize, - self.input_height as usize, - self.input_cv_format, - base_address, - bytes_per_row, - ptr::null(), - ptr::null(), - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreateWithBytes failed: {status}" - ))); - } - - Ok(pixel_buffer) + cv::PixelBuf::with_bytes( + self.input_width as usize, + self.input_height as usize, + base_address, + bytes_per_row, + noop_release, + ptr::null_mut(), + self.input_cv_format, + None, + ) + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferCreateWithBytes: {error}")) + }) } - fn create_output_pixel_buffer(&self) -> Result { - let mut pixel_buffer: CVPixelBufferRef = ptr::null_mut(); - - let status = unsafe { - CVPixelBufferCreate( - ptr::null(), - self.output_width as usize, - self.output_height as usize, - self.output_cv_format, - ptr::null(), - &mut pixel_buffer, - ) - }; - - if status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferCreate failed: {status}" - ))); + fn create_output_pixel_buffer(&self) -> Result, ConvertError> { + if let Some(pool) = &self.pool + && let Ok(buffer) = pool.0.pixel_buf() + { + return Ok(buffer); } - Ok(pixel_buffer) + cv::PixelBuf::new( + self.output_width as usize, + self.output_height as usize, + self.output_cv_format, + None, + ) + .map_err(|error| ConvertError::ConversionFailed(format!("CVPixelBufferCreate: {error}"))) } fn copy_output_to_frame( &self, - pixel_buffer: CVPixelBufferRef, + pixel_buffer: &mut cv::PixelBuf, output: &mut frame::Video, ) -> Result<(), ConvertError> { - unsafe { - let lock_status = CVPixelBufferLockBaseAddress(pixel_buffer, 0); - if lock_status != K_CV_RETURN_SUCCESS { - return Err(ConvertError::ConversionFailed(format!( - "CVPixelBufferLockBaseAddress failed: {lock_status}" - ))); - } - } + unsafe { pixel_buffer.lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY) } + .result() + .map_err(|error| { + ConvertError::ConversionFailed(format!("CVPixelBufferLockBaseAddress: {error}")) + })?; unsafe { - let plane_count = CVPixelBufferGetPlaneCount(pixel_buffer); + let plane_count = pixel_buffer.plane_count(); if plane_count == 0 { let src_ptr = CVPixelBufferGetBaseAddress(pixel_buffer); let src_stride = CVPixelBufferGetBytesPerRow(pixel_buffer); let height = CVPixelBufferGetHeight(pixel_buffer); let dst_stride = output.stride(0); - - let dst_data = output.data_mut(0); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(0).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } else { for plane in 0..plane_count { - let src_ptr = CVPixelBufferGetBaseAddressOfPlane(pixel_buffer, plane); - let src_stride = CVPixelBufferGetBytesPerRowOfPlane(pixel_buffer, plane); - let height = CVPixelBufferGetHeightOfPlane(pixel_buffer, plane); + let src_ptr = pixel_buffer.plane_base_address(plane); + let src_stride = pixel_buffer.plane_bytes_per_row(plane); + let height = pixel_buffer.plane_height(plane); let dst_stride = output.stride(plane); - - let dst_data = output.data_mut(plane); - let dst_ptr = dst_data.as_mut_ptr(); + let dst_ptr = output.data_mut(plane).as_mut_ptr(); if src_stride == dst_stride { ptr::copy_nonoverlapping(src_ptr, dst_ptr, height * src_stride); } else { let copy_len = src_stride.min(dst_stride); for row in 0..height { - let src_row = src_ptr.add(row * src_stride); - let dst_row = dst_ptr.add(row * dst_stride); - ptr::copy_nonoverlapping(src_row, dst_row, copy_len); + ptr::copy_nonoverlapping( + src_ptr.add(row * src_stride), + dst_ptr.add(row * dst_stride), + copy_len, + ); } } } } + } - CVPixelBufferUnlockBaseAddress(pixel_buffer, 0); + unsafe { + let _ = pixel_buffer.unlock_lock_base_addr(cv::pixel_buffer::LockFlags::READ_ONLY); } Ok(()) @@ -283,12 +237,9 @@ impl VideoToolboxConverter { impl Drop for VideoToolboxConverter { fn drop(&mut self) { - let session = self.session.get_mut().0; - if !session.is_null() { - unsafe { - VTPixelTransferSessionInvalidate(session); - CFRelease(session as *const c_void); - } + self.session.get_mut().0.invalidate(); + if let Some(pool) = &self.pool { + pool.0.flush(cv::PixelBufPoolFlushFlags::EXCESS_BUFFERS); } } } @@ -317,25 +268,17 @@ impl FrameConverter for VideoToolboxConverter { } let input_buffer = self.create_input_pixel_buffer(&input)?; - let output_buffer = self.create_output_pixel_buffer()?; - - let status = { - let session_guard = self.session.lock(); - unsafe { - VTPixelTransferSessionTransferImage(session_guard.0, input_buffer, output_buffer) - } - }; + let mut output_buffer = self.create_output_pixel_buffer()?; - unsafe { - CVPixelBufferRelease(input_buffer); - } + let status = self + .session + .lock() + .0 + .transfer(&input_buffer, &output_buffer); - if status != 0 { - unsafe { - CVPixelBufferRelease(output_buffer); - } + if let Err(error) = status { return Err(ConvertError::ConversionFailed(format!( - "VTPixelTransferSessionTransferImage failed: {status}" + "VTPixelTransferSessionTransferImage: {error}" ))); } @@ -345,13 +288,9 @@ impl FrameConverter for VideoToolboxConverter { ); } - self.copy_output_to_frame(output_buffer, output)?; + self.copy_output_to_frame(&mut output_buffer, output)?; output.set_pts(input.pts()); - unsafe { - CVPixelBufferRelease(output_buffer); - } - Ok(()) } diff --git a/crates/recording/examples/camera-upload-bench.rs b/crates/recording/examples/camera-upload-bench.rs new file mode 100644 index 00000000000..692d7858fb0 --- /dev/null +++ b/crates/recording/examples/camera-upload-bench.rs @@ -0,0 +1,179 @@ +#[cfg(windows)] +fn main() -> anyhow::Result<()> { + use cap_camera_ffmpeg::with_media_buffer_data; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_recording::{ + CameraBuffer, CameraBuffers, NativeCameraFrame, create_d3d_device, + upload_mf_buffer_to_texture, + }; + use cap_timestamp::Timestamp; + use std::time::Instant; + use windows::Win32::{ + Graphics::{ + Direct3D11::{ + D3D11_BIND_RENDER_TARGET, D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, + D3D11_TEXTURE2D_DESC, D3D11_USAGE_DEFAULT, ID3D11Resource, + }, + Dxgi::Common::{DXGI_FORMAT_NV12, DXGI_SAMPLE_DESC}, + }, + Media::MediaFoundation::MFCreate2DMediaBuffer, + }; + use windows::core::Interface; + + const WIDTH: usize = 1920; + const HEIGHT: usize = 1080; + const ITERATIONS: usize = 500; + + cap_mediafoundation_utils::thread_init(); + let device = create_d3d_device()?; + let buffer = unsafe { + MFCreate2DMediaBuffer( + WIDTH as u32, + HEIGHT as u32, + u32::from_le_bytes(*b"NV12"), + false, + ) + }?; + + { + let mut lock = buffer.lock_for_write()?; + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 17) as u8; + } + } + + let frame = NativeCameraFrame { + buffer: CameraBuffer::new(buffer), + pixel_format: PixelFormat::NV12, + width: WIDTH as u32, + height: HEIGHT as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(Instant::now()), + }; + let mut buffers = CameraBuffers::new(); + + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut buffers)?; + std::hint::black_box(texture); + } + let elapsed = start.elapsed(); + + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: WIDTH as u32, + Height: HEIGHT as u32, + MipLevels: 1, + ArraySize: 1, + Format: DXGI_FORMAT_NV12, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + + let start = Instant::now(); + with_media_buffer_data( + frame.buffer.get(), + PixelFormat::NV12, + WIDTH, + HEIGHT, + |data, pitch, _| { + for _ in 0..ITERATIONS { + let subresource = D3D11_SUBRESOURCE_DATA { + pSysMem: data.as_ptr() as *const _, + SysMemPitch: pitch as u32, + SysMemSlicePitch: 0, + }; + let mut texture = None; + unsafe { + device.CreateTexture2D( + &texture_desc, + Some(&subresource), + Some(&mut texture), + )?; + } + std::hint::black_box(texture); + } + + Ok::<_, windows::core::Error>(()) + }, + ) + .expect("locked data")?; + let uncached = start.elapsed(); + + let encoder_desc = D3D11_TEXTURE2D_DESC { + BindFlags: (D3D11_BIND_RENDER_TARGET.0 | D3D11_BIND_SHADER_RESOURCE.0) as u32, + ..texture_desc + }; + let mut encoder_texture = None; + unsafe { device.CreateTexture2D(&encoder_desc, None, Some(&mut encoder_texture))? }; + let encoder_texture = encoder_texture.expect("encoder input texture"); + let encoder_resource = encoder_texture.cast::()?; + let context = unsafe { device.GetImmediateContext()? }; + + let mut direct_buffers = CameraBuffers::with_texture(encoder_texture); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut direct_buffers)?; + std::hint::black_box(texture); + } + let direct = start.elapsed(); + + let mut copy_buffers = CameraBuffers::new(); + for _ in 0..20 { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let texture = upload_mf_buffer_to_texture(&device, &frame, &mut copy_buffers)?; + let resource = texture.cast::()?; + unsafe { context.CopyResource(&encoder_resource, &resource) }; + } + let cached_plus_copy = start.elapsed(); + + println!( + "cached upload: {:.3} ms/frame ({:.1} fps)", + elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / elapsed.as_secs_f64() + ); + println!( + "uncached upload: {:.3} ms/frame ({:.1} fps)", + uncached.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / uncached.as_secs_f64() + ); + println!( + "direct upload: {:.3} ms/frame ({:.1} fps)", + direct.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / direct.as_secs_f64() + ); + println!( + "cached + copy: {:.3} ms/frame ({:.1} fps)", + cached_plus_copy.as_secs_f64() * 1000.0 / ITERATIONS as f64, + ITERATIONS as f64 / cached_plus_copy.as_secs_f64() + ); + + Ok(()) +} + +#[cfg(not(windows))] +fn main() { + println!("camera-upload-bench requires Windows"); +} diff --git a/crates/recording/src/feeds/camera.rs b/crates/recording/src/feeds/camera.rs index e70151f06b8..e8b3701cf44 100644 --- a/crates/recording/src/feeds/camera.rs +++ b/crates/recording/src/feeds/camera.rs @@ -25,6 +25,8 @@ use tokio::{sync::oneshot, task::LocalSet}; use tracing::{debug, error, trace, warn}; use crate::ffmpeg::FFmpegVideoFrame; +#[cfg(windows)] +use crate::output_pipeline::CameraBuffer; use crate::output_pipeline::{NativeCameraFrame, VideoFrame}; const CAMERA_INIT_TIMEOUT: Duration = Duration::from_secs(4); @@ -1172,40 +1174,44 @@ async fn setup_camera( .senders .load(std::sync::atomic::Ordering::Relaxed) > 0 - && let Ok(bytes) = frame.native().bytes() { use cap_mediafoundation_utils::IMFMediaBufferExt; use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; - let data_len = bytes.len(); - if let Ok(buffer) = unsafe { MFCreateMemoryBuffer(data_len as u32) } { - let buffer_ready = { - if let Ok(mut lock) = buffer.lock_for_write() { - lock.copy_from_slice(&bytes); - true - } else { - false - } - }; - - if buffer_ready { - let _ = unsafe { buffer.SetCurrentLength(data_len as u32) }; - - #[allow(clippy::arc_with_non_send_sync)] - let buffer = std::sync::Arc::new(std::sync::Mutex::new(buffer)); - hand_native_frame_to_feed( - &native_recipient, - &native_ingress, - NativeCameraFrame { - buffer, - pixel_format: frame.native().pixel_format, - width: frame.native().width as u32, - height: frame.native().height as u32, - is_bottom_up: frame.native().is_bottom_up, - timestamp, - }, - ); - } + let buffer = if let Some(buffer) = frame.native().media_buffer() { + Some(CameraBuffer::new(buffer)) + } else if let Ok(bytes) = frame.native().bytes() { + let data_len = bytes.len(); + unsafe { MFCreateMemoryBuffer(data_len as u32) } + .ok() + .and_then(|buffer| { + buffer + .lock_for_write() + .ok() + .map(|mut lock| lock.copy_from_slice(&bytes)) + .map(|()| buffer) + }) + .and_then(|buffer| { + unsafe { buffer.SetCurrentLength(data_len as u32) }.ok()?; + Some(CameraBuffer::new(buffer)) + }) + } else { + None + }; + + if let Some(buffer) = buffer { + hand_native_frame_to_feed( + &native_recipient, + &native_ingress, + NativeCameraFrame { + buffer, + pixel_format: frame.native().pixel_format, + width: frame.native().width as u32, + height: frame.native().height as u32, + is_bottom_up: frame.native().is_bottom_up, + timestamp, + }, + ); } } diff --git a/crates/recording/src/lib.rs b/crates/recording/src/lib.rs index a7dcbbcc4c9..688a2939190 100644 --- a/crates/recording/src/lib.rs +++ b/crates/recording/src/lib.rs @@ -22,6 +22,8 @@ pub mod upload_preparation; pub mod upload_resume; pub mod upload_verification; +#[cfg(windows)] +pub use capture_pipeline::create_d3d_device; #[cfg(target_os = "linux")] pub use capture_pipeline::target_to_display_and_crop; pub use resolution_limits::{H264_MAX_DIMENSION, calculate_gpu_compatible_size}; diff --git a/crates/recording/src/output_pipeline/win.rs b/crates/recording/src/output_pipeline/win.rs index 8d2fc226df9..bc621e8826f 100644 --- a/crates/recording/src/output_pipeline/win.rs +++ b/crates/recording/src/output_pipeline/win.rs @@ -120,10 +120,13 @@ use windows::{ Foundation::TimeSpan, Graphics::SizeInt32, Win32::Graphics::{ - Direct3D11::ID3D11Device, + Direct3D11::{ + D3D11_TEXTURE2D_DESC, ID3D11Device, ID3D11DeviceContext, ID3D11Resource, + ID3D11Texture2D, + }, Dxgi::Common::{ DXGI_FORMAT, DXGI_FORMAT_B8G8R8A8_UNORM, DXGI_FORMAT_NV12, DXGI_FORMAT_R8G8B8A8_UNORM, - DXGI_FORMAT_YUY2, + DXGI_FORMAT_UNKNOWN, DXGI_FORMAT_YUY2, }, }, }; @@ -524,7 +527,51 @@ impl Muxer for WindowsMuxer { #[cfg(test)] mod tests { - use super::reference_video_frame; + use super::{ + CameraBuffer, MjpegDecoder, NativeCameraFrame, camera_frame_to_ffmpeg, + reference_video_frame, + }; + use cap_camera_windows::PixelFormat; + use cap_mediafoundation_utils::IMFMediaBufferExt; + use cap_timestamp::Timestamp; + use windows::Win32::Media::MediaFoundation::MFCreateMemoryBuffer; + + #[test] + fn native_nv12_frame_converts_without_copying() { + cap_mediafoundation_utils::thread_init(); + + let width = 80; + let height = 48; + let stride = 80; + let len = stride * height + stride * (height / 2); + let buffer = unsafe { MFCreateMemoryBuffer(len as u32) }.expect("buffer"); + { + let mut lock = buffer.lock_for_write().expect("write lock"); + for (index, byte) in lock.iter_mut().enumerate() { + *byte = (index * 13) as u8; + } + } + unsafe { buffer.SetCurrentLength(len as u32) }.expect("length"); + + let base_ptr = { + let lock = buffer.lock().expect("read lock"); + lock.as_ptr() + }; + + let frame = NativeCameraFrame { + buffer: CameraBuffer::new(buffer), + pixel_format: PixelFormat::NV12, + width: width as u32, + height: height as u32, + is_bottom_up: false, + timestamp: Timestamp::Instant(std::time::Instant::now()), + }; + let mut decoder = MjpegDecoder::default(); + let converted = camera_frame_to_ffmpeg(&frame, &mut decoder).expect("conversion"); + + assert_eq!(converted.format(), ffmpeg::format::Pixel::NV12); + assert_eq!(converted.data(0).as_ptr(), base_ptr); + } #[test] fn retained_software_frame_shares_pixels_and_survives_original() { @@ -619,10 +666,29 @@ pub(crate) fn duration_to_timespan(duration: Duration) -> TimeSpan { } } +type CameraMediaBuffer = windows::Win32::Media::MediaFoundation::IMFMediaBuffer; + +#[derive(Clone)] +pub struct CameraBuffer(CameraMediaBuffer); + +impl CameraBuffer { + pub fn new(buffer: CameraMediaBuffer) -> Self { + Self(buffer) + } + + pub fn get(&self) -> &CameraMediaBuffer { + &self.0 + } +} + +// Media Foundation buffers are free-threaded; pixel access is guarded by the +// per-frame 2D or contiguous lock held by the consumer. +unsafe impl Send for CameraBuffer {} +unsafe impl Sync for CameraBuffer {} + #[derive(Clone)] pub struct NativeCameraFrame { - pub buffer: - std::sync::Arc>, + pub buffer: CameraBuffer, pub pixel_format: cap_camera_windows::PixelFormat, pub width: u32, pub height: u32, @@ -896,7 +962,8 @@ impl Muxer for WindowsCameraMuxer { let mut last_frame: Option = None; let mut first_timestamp: Option = None; let mut frame_count = 0u64; - let mut camera_buffers = CameraBuffers::new(); + let mut camera_buffers = + CameraBuffers::with_texture(encoder.input_texture().clone()); let result = encoder.run( Arc::new(AtomicBool::default()), @@ -1116,6 +1183,11 @@ impl AudioMuxer for WindowsCameraMuxer { pub struct CameraBuffers { uyvy_buffer: Vec, flip_buffer: Vec, + texture: Option, + texture_width: u32, + texture_height: u32, + texture_format: DXGI_FORMAT, + context: Option, } impl CameraBuffers { @@ -1123,9 +1195,66 @@ impl CameraBuffers { Self { uyvy_buffer: Vec::new(), flip_buffer: Vec::new(), + texture: None, + texture_width: 0, + texture_height: 0, + texture_format: DXGI_FORMAT_UNKNOWN, + context: None, + } + } + + pub fn with_texture(texture: ID3D11Texture2D) -> Self { + let mut desc = D3D11_TEXTURE2D_DESC::default(); + unsafe { texture.GetDesc(&mut desc) }; + + Self { + texture: Some(texture), + texture_width: desc.Width, + texture_height: desc.Height, + texture_format: desc.Format, + ..Self::new() } } + fn ensure_texture( + &mut self, + device: &ID3D11Device, + desc: &D3D11_TEXTURE2D_DESC, + ) -> windows::core::Result { + if self.texture.is_none() + || self.texture_width != desc.Width + || self.texture_height != desc.Height + || self.texture_format != desc.Format + { + let mut texture = None; + unsafe { device.CreateTexture2D(desc, None, Some(&mut texture))? }; + self.texture = texture; + self.texture_width = desc.Width; + self.texture_height = desc.Height; + self.texture_format = desc.Format; + } + + self.texture.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "CreateTexture2D succeeded but returned no texture", + ) + }) + } + + fn context(&mut self, device: &ID3D11Device) -> windows::core::Result { + if self.context.is_none() { + self.context = Some(unsafe { device.GetImmediateContext()? }); + } + + self.context.clone().ok_or_else(|| { + windows::core::Error::new( + windows::core::HRESULT(-1), + "GetImmediateContext returned no context", + ) + }) + } + fn ensure_uyvy_capacity(&mut self, size: usize) -> &mut [u8] { if self.uyvy_buffer.len() < size { self.uyvy_buffer.resize(size, 0); @@ -1375,6 +1504,19 @@ pub(crate) fn camera_frame_to_ffmpeg( return decode_mjpeg_frame(frame, mjpeg_decoder); } + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) + && let Some(frame) = cap_camera_ffmpeg::media_buffer_as_ffmpeg( + frame.buffer.get(), + frame.pixel_format, + frame.width as usize, + frame.height as usize, + frame.is_bottom_up, + ) + { + cap_camera_ffmpeg::CONVERSION_STATS.record_zero_copy(); + return Ok(frame); + } + let ffmpeg_format = match frame.pixel_format { cap_camera_windows::PixelFormat::NV12 => ffmpeg::format::Pixel::NV12, cap_camera_windows::PixelFormat::YUYV422 => ffmpeg::format::Pixel::YUYV422, @@ -1390,11 +1532,9 @@ pub(crate) fn camera_frame_to_ffmpeg( other => anyhow::bail!("Unsupported camera pixel format: {:?}", other), }; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1654,11 +1794,9 @@ fn decode_mjpeg_frame( ) -> anyhow::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - let buffer_guard = frame + let lock = frame .buffer - .lock() - .map_err(|_| anyhow!("Failed to lock camera buffer"))?; - let lock = buffer_guard + .get() .lock() .map_err(|e| anyhow!("Failed to lock MF buffer: {:?}", e))?; let data = &*lock; @@ -1827,25 +1965,41 @@ pub fn upload_mf_buffer_to_texture( buffers: &mut CameraBuffers, ) -> windows::core::Result { use cap_mediafoundation_utils::IMFMediaBufferExt; - use windows::Win32::Graphics::Direct3D11::{ - D3D11_BIND_SHADER_RESOURCE, D3D11_SUBRESOURCE_DATA, D3D11_TEXTURE2D_DESC, - D3D11_USAGE_DEFAULT, - }; - use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; - let dxgi_format = frame.dxgi_format(); + let buffer = frame.buffer.get(); + + // The zero-copy and preview paths lock these formats with Lock2D; the + // upload must use the same lock type to avoid MF_E_INVALIDREQUEST. + if cap_camera_ffmpeg::should_use_2d_lock(frame.pixel_format, frame.is_bottom_up) + && let Some(texture) = cap_camera_ffmpeg::with_media_buffer_data( + buffer, + frame.pixel_format, + frame.width as usize, + frame.height as usize, + |data, primary_stride, secondary_stride| { + upload_mf_buffer_data_to_texture( + device, + frame, + data, + CameraBufferLayout { + primary_stride, + secondary_stride, + }, + buffers, + ) + }, + ) + { + return texture; + } - let buffer_guard = frame - .buffer - .lock() - .map_err(|_| windows::core::Error::from(windows::core::HRESULT(-1)))?; - let lock = buffer_guard.lock()?; - let original_data = &*lock; + let lock = buffer.lock()?; + let data = &*lock; let layout = infer_camera_buffer_layout( frame.pixel_format, frame.width as usize, frame.height as usize, - original_data.len(), + data.len(), ) .ok_or_else(|| { windows::core::Error::new( @@ -1855,11 +2009,44 @@ pub fn upload_mf_buffer_to_texture( frame.pixel_format, frame.width, frame.height, - original_data.len() + data.len() ), ) })?; + upload_mf_buffer_data_to_texture(device, frame, data, layout, buffers) +} + +fn upload_mf_buffer_data_to_texture( + device: &ID3D11Device, + frame: &NativeCameraFrame, + original_data: &[u8], + layout: CameraBufferLayout, + buffers: &mut CameraBuffers, +) -> windows::core::Result { + use windows::Win32::Graphics::Direct3D11::{D3D11_BIND_SHADER_RESOURCE, D3D11_USAGE_DEFAULT}; + use windows::Win32::Graphics::Dxgi::Common::DXGI_SAMPLE_DESC; + use windows::core::Interface; + + let dxgi_format = frame.dxgi_format(); + let texture_desc = D3D11_TEXTURE2D_DESC { + Width: frame.width, + Height: frame.height, + MipLevels: 1, + ArraySize: 1, + Format: dxgi_format, + SampleDesc: DXGI_SAMPLE_DESC { + Count: 1, + Quality: 0, + }, + Usage: D3D11_USAGE_DEFAULT, + BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, + CPUAccessFlags: 0, + MiscFlags: 0, + }; + let texture: ID3D11Texture2D = buffers.ensure_texture(device, &texture_desc)?; + let context: ID3D11DeviceContext = buffers.context(device)?; + let needs_uyvy_conversion = frame.pixel_format == cap_camera_windows::PixelFormat::UYVY422; let needs_flip = frame.is_bottom_up; let mut row_pitch = layout.primary_stride as u32; @@ -1944,36 +2131,11 @@ pub fn upload_mf_buffer_to_texture( } }; - let texture_desc = D3D11_TEXTURE2D_DESC { - Width: frame.width, - Height: frame.height, - MipLevels: 1, - ArraySize: 1, - Format: dxgi_format, - SampleDesc: DXGI_SAMPLE_DESC { - Count: 1, - Quality: 0, - }, - Usage: D3D11_USAGE_DEFAULT, - BindFlags: D3D11_BIND_SHADER_RESOURCE.0 as u32, - CPUAccessFlags: 0, - MiscFlags: 0, - }; - - let subresource_data = D3D11_SUBRESOURCE_DATA { - pSysMem: data.as_ptr() as *const _, - SysMemPitch: row_pitch, - SysMemSlicePitch: 0, - }; + let resource = texture.cast::()?; unsafe { - let mut texture = None; - device.CreateTexture2D(&texture_desc, Some(&subresource_data), Some(&mut texture))?; - texture.ok_or_else(|| { - windows::core::Error::new( - windows::core::HRESULT(-1), - "CreateTexture2D succeeded but returned no texture", - ) - }) + context.UpdateSubresource(&resource, 0, None, data.as_ptr() as *const _, row_pitch, 0); } + + Ok(texture) } diff --git a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs index 4e61d2300ed..97fbd0cf402 100644 --- a/crates/recording/src/output_pipeline/win_fragmented_m4s.rs +++ b/crates/recording/src/output_pipeline/win_fragmented_m4s.rs @@ -1100,7 +1100,9 @@ impl WindowsFragmentedM4SCameraMuxer { } let mut pacer = FramePacer::new(video_config.fps()); - let mut last_frame: Option = None; + // Repeats share the last converted frame so the capture buffer + // is locked once instead of on every paced repeat. + let mut last_ffmpeg_frame: Option = None; let mut first_timestamp: Option = None; let mut slow_convert_count = 0u32; @@ -1122,17 +1124,14 @@ impl WindowsFragmentedM4SCameraMuxer { } }; - let encode_camera_frame_fn = + let convert_camera_frame = |camera_frame: &NativeCameraFrame, - timestamp: Duration, slow_convert_count: &mut u32, - slow_encode_count: &mut u32, - total_frames: &mut u64, - encoder: &Arc>, mjpeg_decoder: &mut MjpegDecoder| - -> anyhow::Result<()> { + -> anyhow::Result { let convert_start = std::time::Instant::now(); - let ffmpeg_frame_result = camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); + let ffmpeg_frame_result = + camera_frame_to_ffmpeg(camera_frame, mjpeg_decoder); let convert_elapsed_ms = convert_start.elapsed().as_millis(); if convert_elapsed_ms > SLOW_THRESHOLD_MS { @@ -1147,41 +1146,42 @@ impl WindowsFragmentedM4SCameraMuxer { } } - match ffmpeg_frame_result { - Ok(ffmpeg_frame) => { - let encode_start = std::time::Instant::now(); + ffmpeg_frame_result + }; - match encoder.lock() { - Ok(mut enc) => { - if let Err(e) = enc.queue_frame(ffmpeg_frame, timestamp) { - warn!("Failed to encode camera frame: {e}"); - } - } - Err(_) => { - return Err(anyhow!( - "Camera encoder mutex poisoned - all subsequent frames would be lost" - )); - } - } + let encode_camera_frame_fn = + |ffmpeg_frame: ffmpeg::frame::Video, + timestamp: Duration, + slow_encode_count: &mut u32, + total_frames: &mut u64, + encoder: &Arc>| + -> anyhow::Result<()> { + let encode_start = std::time::Instant::now(); - let encode_elapsed_ms = encode_start.elapsed().as_millis(); - - if encode_elapsed_ms > SLOW_THRESHOLD_MS { - *slow_encode_count += 1; - if *slow_encode_count <= 5 - || slow_encode_count.is_multiple_of(100) - { - debug!( - elapsed_ms = encode_elapsed_ms, - count = *slow_encode_count, - "Camera encoder.queue_frame exceeded {}ms threshold", - SLOW_THRESHOLD_MS - ); - } + match encoder.lock() { + Ok(mut enc) => { + if let Err(e) = enc.queue_frame(ffmpeg_frame, timestamp) { + warn!("Failed to encode camera frame: {e}"); } } - Err(e) => { - warn!("Failed to convert camera frame: {e:?}"); + Err(_) => { + return Err(anyhow!( + "Camera encoder mutex poisoned - all subsequent frames would be lost" + )); + } + } + + let encode_elapsed_ms = encode_start.elapsed().as_millis(); + + if encode_elapsed_ms > SLOW_THRESHOLD_MS { + *slow_encode_count += 1; + if *slow_encode_count <= 5 || slow_encode_count.is_multiple_of(100) { + debug!( + elapsed_ms = encode_elapsed_ms, + count = *slow_encode_count, + "Camera encoder.queue_frame exceeded {}ms threshold", + SLOW_THRESHOLD_MS + ); } } @@ -1201,17 +1201,30 @@ impl WindowsFragmentedM4SCameraMuxer { break; } - let (camera_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { - Paced::Frame(frame, ts) => { - last_frame = Some(frame.clone()); - (frame, ts) - } + let (ffmpeg_frame, timestamp) = match pacer.next(&video_rx, &pause_flag) { + Paced::Frame(frame, ts) => match convert_camera_frame( + &frame, + &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => { + last_ffmpeg_frame = Some(reference_video_frame(&ffmpeg_frame)); + (ffmpeg_frame, ts) + } + Err(e) => { + warn!("Failed to convert camera frame: {e:?}"); + let Some(last) = last_ffmpeg_frame.as_ref() else { + continue; + }; + (reference_video_frame(last), ts) + } + }, Paced::Repeat(ts) => { - let Some(frame) = &last_frame else { + let Some(last) = last_ffmpeg_frame.as_ref() else { continue; }; duplicated_frames += 1; - (frame.clone(), ts) + (reference_video_frame(last), ts) } Paced::End => { let mut drained = 0u64; @@ -1223,14 +1236,25 @@ impl WindowsFragmentedM4SCameraMuxer { Ok(Some((frame, ts))) => { let normalized_ts = normalize_timestamp(ts, &mut first_timestamp); - if let Err(e) = encode_camera_frame_fn( + let ffmpeg_frame = match convert_camera_frame( &frame, - normalized_ts, &mut slow_convert_count, + &mut mjpeg_decoder, + ) { + Ok(ffmpeg_frame) => ffmpeg_frame, + Err(e) => { + warn!( + "Failed to convert drained camera frame: {e:?}" + ); + continue; + } + }; + if let Err(e) = encode_camera_frame_fn( + ffmpeg_frame, + normalized_ts, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { warn!("Failed to encode drained camera frame: {e}"); break; @@ -1259,13 +1283,11 @@ impl WindowsFragmentedM4SCameraMuxer { let normalized_ts = normalize_timestamp(timestamp, &mut first_timestamp); if let Err(e) = encode_camera_frame_fn( - &camera_frame, + ffmpeg_frame, normalized_ts, - &mut slow_convert_count, &mut slow_encode_count, &mut total_frames, &encoder_clone, - &mut mjpeg_decoder, ) { error!("Camera encoder mutex poisoned - encoder thread likely panicked, stopping"); return Err(e); diff --git a/crates/verify/tests/recording.rs b/crates/verify/tests/recording.rs index 61abf3a06b4..ffb3d26e8e7 100644 --- a/crates/verify/tests/recording.rs +++ b/crates/verify/tests/recording.rs @@ -31,6 +31,8 @@ use cap_project::{ InstantRecordingMeta, ProjectConfiguration, RecordingMeta, RecordingMetaInner, StudioRecordingMeta, XY, }; +#[cfg(windows)] +use cap_recording::CameraBuffer; use cap_recording::{ NativeCameraFrame, RecordingDefaults, StudioQuality, feeds::{ @@ -1134,10 +1136,8 @@ fn camera_frame(slot: u32, capture: u64) -> Option { let buffer = unsafe { MFCreateMemoryBuffer(nv12.len() as u32) }.ok()?; buffer.lock_for_write().ok()?.copy_from_slice(&nv12); unsafe { buffer.SetCurrentLength(nv12.len() as u32) }.ok()?; - #[allow(clippy::arc_with_non_send_sync)] - let buffer = Arc::new(std::sync::Mutex::new(buffer)); Some(NativeCameraFrame { - buffer, + buffer: CameraBuffer::new(buffer), pixel_format: cap_camera_windows::PixelFormat::NV12, width: w as u32, height: h as u32,