From 39016e9635fe3ec2c04ba63daeaa10054e12de7c Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sat, 24 Oct 2026 22:26:10 +0000 Subject: [PATCH 1/2] improve: copy Linux camera frames once --- crates/camera-ffmpeg/src/linux.rs | 324 +++++++++++++++++++++-------- crates/camera/examples/cli.rs | 2 +- crates/camera/examples/diagnose.rs | 7 + crates/camera/src/lib.rs | 41 +++- crates/camera/src/linux.rs | 8 +- 5 files changed, 292 insertions(+), 90 deletions(-) diff --git a/crates/camera-ffmpeg/src/linux.rs b/crates/camera-ffmpeg/src/linux.rs index 89e42e2a85b..ad28a4be34e 100644 --- a/crates/camera-ffmpeg/src/linux.rs +++ b/crates/camera-ffmpeg/src/linux.rs @@ -72,24 +72,85 @@ fn decode_mjpeg(bytes: &[u8]) -> Result { }) } -impl CapturedFrameExt for CapturedFrame { +impl CapturedFrameExt for CapturedFrame<'_> { fn as_ffmpeg(&self) -> Result { let native = self.native(); - let format = native.format; - let bytes = native.bytes.as_slice(); - - match fourcc_str(format.fourcc).as_str() { - "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), - "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), - "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), - "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3), - "NV12" => copy_nv12(bytes, format), - "YU12" => copy_yuv420(bytes, format, false), - "YV12" => copy_yuv420(bytes, format, true), - "MJPG" | "JPEG" => decode_mjpeg(bytes), - other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), - } + frame_from_bytes(native.bytes, native.format) + } +} + +fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result { + match fourcc_str(format.fourcc).as_str() { + "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2), + "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2), + "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3), + "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3), + "NV12" => copy_nv12(bytes, format), + "YU12" => copy_yuv420(bytes, format, false), + "YV12" => copy_yuv420(bytes, format, true), + "MJPG" | "JPEG" => decode_mjpeg(bytes), + other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())), + } +} + +const FRAME_POOL_CAPACITY: usize = 4; + +struct FramePool { + frames: Vec, +} + +thread_local! { + static FRAME_POOL: RefCell = const { RefCell::new(FramePool { frames: Vec::new() }) }; +} + +fn with_pooled_frame( + pixel: Pixel, + width: u32, + height: u32, + fill: impl FnOnce(&mut FFVideo) -> Result<(), AsFFmpegError>, +) -> Result { + FRAME_POOL.with(|pool| { + let mut pool = pool.borrow_mut(); + let index = if let Some(index) = matching_reusable(&pool.frames, pixel, width, height) { + index + } else if let Some(index) = pool.frames.iter().position(frame_is_reusable) { + pool.frames[index] = FFVideo::new(pixel, width, height); + index + } else if pool.frames.len() < FRAME_POOL_CAPACITY { + pool.frames.push(FFVideo::new(pixel, width, height)); + pool.frames.len() - 1 + } else { + let mut frame = FFVideo::new(pixel, width, height); + fill(&mut frame)?; + return Ok(frame); + }; + fill(&mut pool.frames[index])?; + Ok(reference_frame(&pool.frames[index])) + }) +} + +fn matching_reusable(frames: &[FFVideo], pixel: Pixel, width: u32, height: u32) -> Option { + frames.iter().position(|frame| { + frame.format() == pixel + && frame.width() == width + && frame.height() == height + && frame_is_reusable(frame) + }) +} + +fn frame_is_reusable(frame: &FFVideo) -> bool { + // av_frame_ref hands this buffer to the feed. It can be filled again only + // after those references are gone, or a live frame would change underneath. + unsafe { ffmpeg::ffi::av_frame_is_writable(frame.as_ptr().cast_mut()) != 0 } +} + +fn reference_frame(frame: &FFVideo) -> FFVideo { + let mut referenced = FFVideo::empty(); + let status = unsafe { ffmpeg::ffi::av_frame_ref(referenced.as_mut_ptr(), frame.as_ptr()) }; + if status < 0 { + return frame.clone(); } + referenced } fn copy_packed( @@ -104,18 +165,18 @@ fn copy_packed( let source_stride = format.stride.max(row_width); require_len(bytes, source_stride * height, format)?; - let mut frame = FFVideo::new(pixel, format.width, format.height); - let dest_stride = frame.stride(0); - let dest = frame.data_mut(0); - - for y in 0..height { - let source_start = y * source_stride; - let source_end = source_start + row_width; - let dest_start = y * dest_stride; - dest[dest_start..dest_start + row_width].copy_from_slice(&bytes[source_start..source_end]); - } - - Ok(frame) + with_pooled_frame(pixel, format.width, format.height, |frame| { + let dest_stride = frame.stride(0); + let dest = frame.data_mut(0); + for y in 0..height { + let source_start = y * source_stride; + let source_end = source_start + row_width; + let dest_start = y * dest_stride; + dest[dest_start..dest_start + row_width] + .copy_from_slice(&bytes[source_start..source_end]); + } + Ok(()) + }) } fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result { @@ -126,29 +187,29 @@ fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result String { String::from_utf8_lossy(&fourcc).to_string() } + +#[cfg(test)] +mod tests { + use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes}; + use cap_camera::NativeFrameFormat; + use ffmpeg::format::Pixel; + + fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat { + NativeFrameFormat { + fourcc: *fourcc, + width, + height, + stride, + } + } + + fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec { + let stride = frame.stride(plane); + let height = frame.plane_height(plane) as usize; + let data = frame.data(plane); + let mut rows = Vec::with_capacity(height * row_width); + for y in 0..height { + let start = y * stride; + rows.extend_from_slice(&data[start..start + row_width]); + } + rows + } + + #[test] + fn packed_yuyv_keeps_rows_and_drops_source_padding() { + let _ = ffmpeg::init(); + let source = [1, 2, 3, 4, 9, 9, 5, 6, 7, 8, 8, 8]; + let frame = copy_packed(&source, format(b"YUYV", 2, 2, 6), Pixel::YUYV422, 2).unwrap(); + assert_eq!(frame.format(), Pixel::YUYV422); + assert_eq!(frame.width(), 2); + assert_eq!(frame.height(), 2); + assert_eq!(active_rows(&frame, 0, 4), vec![1, 2, 3, 4, 5, 6, 7, 8]); + } + + #[test] + fn nv12_copies_y_and_interleaved_chroma() { + let _ = ffmpeg::init(); + let source = [1, 2, 3, 4, 5, 6]; + let frame = copy_nv12(&source, format(b"NV12", 2, 2, 2)).unwrap(); + assert_eq!(active_rows(&frame, 0, 2), vec![1, 2, 3, 4]); + assert_eq!(active_rows(&frame, 1, 2), vec![5, 6]); + } + + #[test] + fn yv12_stores_v_before_u() { + let _ = ffmpeg::init(); + let source = [1, 2, 3, 4, 9, 8]; + let yv12 = frame_from_bytes(&source, format(b"YV12", 2, 2, 2)).unwrap(); + let yu12 = frame_from_bytes(&source, format(b"YU12", 2, 2, 2)).unwrap(); + assert_eq!(active_rows(&yv12, 0, 2), vec![1, 2, 3, 4]); + assert_eq!(active_rows(&yv12, 1, 1), vec![8]); + assert_eq!(active_rows(&yv12, 2, 1), vec![9]); + assert_eq!(active_rows(&yu12, 1, 1), vec![9]); + assert_eq!(active_rows(&yu12, 2, 1), vec![8]); + } + + #[test] + fn a_live_frame_keeps_its_pixels_when_the_next_one_is_filled() { + let _ = ffmpeg::init(); + let first = + copy_packed(&[1, 2, 3, 4], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap(); + let second = + copy_packed(&[5, 6, 7, 8], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap(); + assert_eq!(active_rows(&first, 0, 4), vec![1, 2, 3, 4]); + assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]); + drop(first); + let third = copy_packed( + &[9, 10, 11, 12], + format(b"YUYV", 2, 1, 4), + Pixel::YUYV422, + 2, + ) + .unwrap(); + assert_eq!(active_rows(&third, 0, 4), vec![9, 10, 11, 12]); + assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]); + } + + #[test] + fn short_and_odd_frames_keep_their_errors() { + let _ = ffmpeg::init(); + let short = copy_packed(&[1], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2); + assert!(matches!(short, Err(AsFFmpegError::InvalidFrameData { .. }))); + let odd = frame_from_bytes(&[1, 2, 3, 4, 5, 6], format(b"YU12", 2, 2, 3)); + assert!(matches!(odd, Err(AsFFmpegError::InvalidFrameStride { .. }))); + let unsupported = frame_from_bytes(&[], format(b"XXXX", 1, 1, 1)); + assert!(matches!( + unsupported, + Err(AsFFmpegError::UnsupportedFormat(_)) + )); + } +} diff --git a/crates/camera/examples/cli.rs b/crates/camera/examples/cli.rs index 8e7bc1385f9..eb80293d933 100644 --- a/crates/camera/examples/cli.rs +++ b/crates/camera/examples/cli.rs @@ -27,7 +27,7 @@ fn main() { let _handle = selected_camera .start_capturing(selected_format.0, |frame| { - drop(frame); + let _ = frame.native().bytes.len(); }) .unwrap(); diff --git a/crates/camera/examples/diagnose.rs b/crates/camera/examples/diagnose.rs index 8abf734a3ab..67866b3eb30 100644 --- a/crates/camera/examples/diagnose.rs +++ b/crates/camera/examples/diagnose.rs @@ -196,6 +196,13 @@ fn capture_test(camera: &CameraInfo, mode: CaptureMode) { } } +#[cfg(target_os = "linux")] +fn describe_frame(frame: &cap_camera::CapturedFrame<'_>) -> String { + let _ = frame; + "".to_string() +} + +#[cfg(not(target_os = "linux"))] fn describe_frame(frame: &cap_camera::CapturedFrame) -> String { #[cfg(target_os = "macos")] { diff --git a/crates/camera/src/lib.rs b/crates/camera/src/lib.rs index 98b2875060b..dd160277749 100644 --- a/crates/camera/src/lib.rs +++ b/crates/camera/src/lib.rs @@ -218,6 +218,7 @@ pub enum CaptureMode { Compatibility, } +#[cfg(not(target_os = "linux"))] #[derive(Debug)] pub struct CapturedFrame { native: NativeCapturedFrame, @@ -226,17 +227,35 @@ pub struct CapturedFrame { // pub capture_begin_time: Option, } +#[cfg(target_os = "linux")] +#[derive(Debug)] +pub struct CapturedFrame<'a> { + native: NativeCapturedFrame<'a>, + // pub reference_time: Instant, + pub timestamp: Duration, + // pub capture_begin_time: Option, +} + +#[cfg(not(target_os = "linux"))] impl CapturedFrame { pub fn native(&self) -> &NativeCapturedFrame { &self.native } } +#[cfg(target_os = "linux")] +impl<'a> CapturedFrame<'a> { + pub fn native(&self) -> &NativeCapturedFrame<'a> { + &self.native + } +} + impl CameraInfo { pub fn formats(&self) -> Option> { self.formats_impl() } + #[cfg(not(target_os = "linux"))] pub fn start_capturing( &self, format: Format, @@ -245,6 +264,16 @@ impl CameraInfo { self.start_capturing_with_mode(format, CaptureMode::default(), callback) } + #[cfg(target_os = "linux")] + pub fn start_capturing( + &self, + format: Format, + callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static, + ) -> Result { + self.start_capturing_with_mode(format, CaptureMode::default(), callback) + } + + #[cfg(not(target_os = "linux"))] #[cfg_attr(not(target_os = "macos"), allow(unused_variables))] pub fn start_capturing_with_mode( &self, @@ -262,7 +291,17 @@ impl CameraInfo { )?), #[cfg(windows)] native: Some(start_capturing_impl(self, format, Box::new(callback))?), - #[cfg(target_os = "linux")] + }) + } + + #[cfg(target_os = "linux")] + pub fn start_capturing_with_mode( + &self, + format: Format, + _mode: CaptureMode, + callback: impl for<'a> FnMut(CapturedFrame<'a>) + Send + 'static, + ) -> Result { + Ok(CaptureHandle { native: Some(start_capturing_impl(self, format, Box::new(callback))?), }) } diff --git a/crates/camera/src/linux.rs b/crates/camera/src/linux.rs index f12147c1067..16b3bba236e 100644 --- a/crates/camera/src/linux.rs +++ b/crates/camera/src/linux.rs @@ -63,8 +63,8 @@ pub struct NativeFrameFormat { } #[derive(Debug)] -pub struct NativeCapturedFrame { - pub bytes: Vec, +pub struct NativeCapturedFrame<'a> { + pub bytes: &'a [u8], pub format: NativeFrameFormat, } @@ -183,7 +183,7 @@ impl CameraInfo { pub fn start_capturing_impl( camera: &CameraInfo, format: Format, - mut callback: Box, + mut callback: Box FnMut(CapturedFrame<'a>) + Send>, ) -> Result { let device = open_device(camera)?; let requested = V4lFormat::new( @@ -252,7 +252,7 @@ pub fn start_capturing_impl( let bytes = bytes.get(..used).unwrap_or(bytes); callback(CapturedFrame { native: NativeCapturedFrame { - bytes: bytes.to_vec(), + bytes, format: frame_format, }, timestamp: started.elapsed(), From 593890d7562b9105ef4c857b6bd942b58ae9e73d Mon Sep 17 00:00:00 2001 From: Kyzer <36684609+kyzer1023@users.noreply.github.com> Date: Sun, 25 Oct 2026 03:32:08 +0000 Subject: [PATCH 2/2] fix: keep camera CLI example platform-neutral --- crates/camera/examples/cli.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/camera/examples/cli.rs b/crates/camera/examples/cli.rs index eb80293d933..c0e62491ff7 100644 --- a/crates/camera/examples/cli.rs +++ b/crates/camera/examples/cli.rs @@ -26,9 +26,7 @@ fn main() { .unwrap(); let _handle = selected_camera - .start_capturing(selected_format.0, |frame| { - let _ = frame.native().bytes.len(); - }) + .start_capturing(selected_format.0, |_frame| {}) .unwrap(); std::thread::sleep(Duration::from_secs(5));