perf(linux-camera): copy frames once

crates/camera-ffmpeg/src/linux.rs

@@ -72,24 +72,85 @@ fn decode_mjpeg(bytes: &[u8]) -> Result<FFVideo, AsFFmpegError> {
     })
 }
 
-impl CapturedFrameExt for CapturedFrame {
+impl CapturedFrameExt for CapturedFrame<'_> {
     fn as_ffmpeg(&self) -> Result<FFVideo, AsFFmpegError> {
         let native = self.native();
-        let format = native.format;
-        let bytes = native.bytes.as_slice();
-
-        match fourcc_str(format.fourcc).as_str() {
-            "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),
-            "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),
-            "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),
-            "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),
-            "NV12" => copy_nv12(bytes, format),
-            "YU12" => copy_yuv420(bytes, format, false),
-            "YV12" => copy_yuv420(bytes, format, true),
-            "MJPG" | "JPEG" => decode_mjpeg(bytes),
-            other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),
-        }
+        frame_from_bytes(native.bytes, native.format)
+    }
+}
+
+fn frame_from_bytes(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {
+    match fourcc_str(format.fourcc).as_str() {
+        "YUYV" => copy_packed(bytes, format, Pixel::YUYV422, 2),
+        "UYVY" => copy_packed(bytes, format, Pixel::UYVY422, 2),
+        "RGB3" => copy_packed(bytes, format, Pixel::RGB24, 3),
+        "BGR3" => copy_packed(bytes, format, Pixel::BGR24, 3),
+        "NV12" => copy_nv12(bytes, format),
+        "YU12" => copy_yuv420(bytes, format, false),
+        "YV12" => copy_yuv420(bytes, format, true),
+        "MJPG" | "JPEG" => decode_mjpeg(bytes),
+        other => Err(AsFFmpegError::UnsupportedFormat(other.to_string())),
+    }
+}
+
+const FRAME_POOL_CAPACITY: usize = 4;
+
+struct FramePool {
+    frames: Vec<FFVideo>,
+}
+
+thread_local! {
+    static FRAME_POOL: RefCell<FramePool> = const { RefCell::new(FramePool { frames: Vec::new() }) };
+}
+
+fn with_pooled_frame(
+    pixel: Pixel,
+    width: u32,
+    height: u32,
+    fill: impl FnOnce(&mut FFVideo) -> Result<(), AsFFmpegError>,
+) -> Result<FFVideo, AsFFmpegError> {
+    FRAME_POOL.with(|pool| {
+        let mut pool = pool.borrow_mut();
+        let index = if let Some(index) = matching_reusable(&pool.frames, pixel, width, height) {
+            index
+        } else if let Some(index) = pool.frames.iter().position(frame_is_reusable) {
+            pool.frames[index] = FFVideo::new(pixel, width, height);
+            index
+        } else if pool.frames.len() < FRAME_POOL_CAPACITY {
+            pool.frames.push(FFVideo::new(pixel, width, height));
+            pool.frames.len() - 1
+        } else {
+            let mut frame = FFVideo::new(pixel, width, height);
+            fill(&mut frame)?;
+            return Ok(frame);
+        };
+        fill(&mut pool.frames[index])?;
+        Ok(reference_frame(&pool.frames[index]))
+    })
+}
+
+fn matching_reusable(frames: &[FFVideo], pixel: Pixel, width: u32, height: u32) -> Option<usize> {
+    frames.iter().position(|frame| {
+        frame.format() == pixel
+            && frame.width() == width
+            && frame.height() == height
+            && frame_is_reusable(frame)
+    })
+}
+
+fn frame_is_reusable(frame: &FFVideo) -> bool {
+    // av_frame_ref hands this buffer to the feed. It can be filled again only
+    // after those references are gone, or a live frame would change underneath.
+    unsafe { ffmpeg::ffi::av_frame_is_writable(frame.as_ptr().cast_mut()) != 0 }
+}
+
+fn reference_frame(frame: &FFVideo) -> FFVideo {
+    let mut referenced = FFVideo::empty();
+    let status = unsafe { ffmpeg::ffi::av_frame_ref(referenced.as_mut_ptr(), frame.as_ptr()) };
+    if status < 0 {
+        return frame.clone();
     }
+    referenced
 }
 
 fn copy_packed(
@@ -104,18 +165,18 @@ fn copy_packed(
     let source_stride = format.stride.max(row_width);
     require_len(bytes, source_stride * height, format)?;
 
-    let mut frame = FFVideo::new(pixel, format.width, format.height);
-    let dest_stride = frame.stride(0);
-    let dest = frame.data_mut(0);
-
-    for y in 0..height {
-        let source_start = y * source_stride;
-        let source_end = source_start + row_width;
-        let dest_start = y * dest_stride;
-        dest[dest_start..dest_start + row_width].copy_from_slice(&bytes[source_start..source_end]);
-    }
-
-    Ok(frame)
+    with_pooled_frame(pixel, format.width, format.height, |frame| {
+        let dest_stride = frame.stride(0);
+        let dest = frame.data_mut(0);
+        for y in 0..height {
+            let source_start = y * source_stride;
+            let source_end = source_start + row_width;
+            let dest_start = y * dest_stride;
+            dest[dest_start..dest_start + row_width]
+                .copy_from_slice(&bytes[source_start..source_end]);
+        }
+        Ok(())
+    })
 }
 
 fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpegError> {
@@ -126,29 +187,29 @@ fn copy_nv12(bytes: &[u8], format: NativeFrameFormat) -> Result<FFVideo, AsFFmpe
     let uv_height = height / 2;
     require_len(bytes, y_size + source_stride * uv_height, format)?;
 
-    let mut frame = FFVideo::new(Pixel::NV12, format.width, format.height);
-    let y_dest_stride = frame.stride(0);
-    copy_plane(
-        bytes,
-        0,
-        source_stride,
-        frame.data_mut(0),
-        y_dest_stride,
-        width,
-        height,
-    );
-    let uv_dest_stride = frame.stride(1);
-    copy_plane(
-        bytes,
-        y_size,
-        source_stride,
-        frame.data_mut(1),
-        uv_dest_stride,
-        width,
-        uv_height,
-    );
-
-    Ok(frame)
+    with_pooled_frame(Pixel::NV12, format.width, format.height, |frame| {
+        let y_dest_stride = frame.stride(0);
+        copy_plane(
+            bytes,
+            0,
+            source_stride,
+            frame.data_mut(0),
+            y_dest_stride,
+            width,
+            height,
+        );
+        let uv_dest_stride = frame.stride(1);
+        copy_plane(
+            bytes,
+            y_size,
+            source_stride,
+            frame.data_mut(1),
+            uv_dest_stride,
+            width,
+            uv_height,
+        );
+        Ok(())
+    })
 }
 
 fn copy_yuv420(
@@ -172,43 +233,42 @@ fn copy_yuv420(
     let chroma_size = chroma_stride * chroma_height;
     require_len(bytes, y_size + chroma_size * 2, format)?;
 
-    let mut frame = FFVideo::new(Pixel::YUV420P, format.width, format.height);
-    let y_dest_stride = frame.stride(0);
-    copy_plane(
-        bytes,
-        0,
-        y_stride,
-        frame.data_mut(0),
-        y_dest_stride,
-        width,
-        height,
-    );
-
     let first_chroma_plane = if v_before_u { 2 } else { 1 };
     let second_chroma_plane = if v_before_u { 1 } else { 2 };
 
-    let first_chroma_stride = frame.stride(first_chroma_plane);
-    copy_plane(
-        bytes,
-        y_size,
-        chroma_stride,
-        frame.data_mut(first_chroma_plane),
-        first_chroma_stride,
-        chroma_width,
-        chroma_height,
-    );
-    let second_chroma_stride = frame.stride(second_chroma_plane);
-    copy_plane(
-        bytes,
-        y_size + chroma_size,
-        chroma_stride,
-        frame.data_mut(second_chroma_plane),
-        second_chroma_stride,
-        chroma_width,
-        chroma_height,
-    );
-
-    Ok(frame)
+    with_pooled_frame(Pixel::YUV420P, format.width, format.height, |frame| {
+        let y_dest_stride = frame.stride(0);
+        copy_plane(
+            bytes,
+            0,
+            y_stride,
+            frame.data_mut(0),
+            y_dest_stride,
+            width,
+            height,
+        );
+        let first_chroma_stride = frame.stride(first_chroma_plane);
+        copy_plane(
+            bytes,
+            y_size,
+            chroma_stride,
+            frame.data_mut(first_chroma_plane),
+            first_chroma_stride,
+            chroma_width,
+            chroma_height,
+        );
+        let second_chroma_stride = frame.stride(second_chroma_plane);
+        copy_plane(
+            bytes,
+            y_size + chroma_size,
+            chroma_stride,
+            frame.data_mut(second_chroma_plane),
+            second_chroma_stride,
+            chroma_width,
+            chroma_height,
+        );
+        Ok(())
+    })
 }
 
 fn copy_plane(
@@ -247,3 +307,99 @@ fn require_len(
 fn fourcc_str(fourcc: [u8; 4]) -> String {
     String::from_utf8_lossy(&fourcc).to_string()
 }
+
+#[cfg(test)]
+mod tests {
+    use super::{AsFFmpegError, copy_nv12, copy_packed, frame_from_bytes};
+    use cap_camera::NativeFrameFormat;
+    use ffmpeg::format::Pixel;
+
+    fn format(fourcc: &[u8; 4], width: u32, height: u32, stride: usize) -> NativeFrameFormat {
+        NativeFrameFormat {
+            fourcc: *fourcc,
+            width,
+            height,
+            stride,
+        }
+    }
+
+    fn active_rows(frame: &ffmpeg::frame::Video, plane: usize, row_width: usize) -> Vec<u8> {
+        let stride = frame.stride(plane);
+        let height = frame.plane_height(plane) as usize;
+        let data = frame.data(plane);
+        let mut rows = Vec::with_capacity(height * row_width);
+        for y in 0..height {
+            let start = y * stride;
+            rows.extend_from_slice(&data[start..start + row_width]);
+        }
+        rows
+    }
+
+    #[test]
+    fn packed_yuyv_keeps_rows_and_drops_source_padding() {
+        let _ = ffmpeg::init();
+        let source = [1, 2, 3, 4, 9, 9, 5, 6, 7, 8, 8, 8];
+        let frame = copy_packed(&source, format(b"YUYV", 2, 2, 6), Pixel::YUYV422, 2).unwrap();
+        assert_eq!(frame.format(), Pixel::YUYV422);
+        assert_eq!(frame.width(), 2);
+        assert_eq!(frame.height(), 2);
+        assert_eq!(active_rows(&frame, 0, 4), vec![1, 2, 3, 4, 5, 6, 7, 8]);
+    }
+
+    #[test]
+    fn nv12_copies_y_and_interleaved_chroma() {
+        let _ = ffmpeg::init();
+        let source = [1, 2, 3, 4, 5, 6];
+        let frame = copy_nv12(&source, format(b"NV12", 2, 2, 2)).unwrap();
+        assert_eq!(active_rows(&frame, 0, 2), vec![1, 2, 3, 4]);
+        assert_eq!(active_rows(&frame, 1, 2), vec![5, 6]);
+    }
+
+    #[test]
+    fn yv12_stores_v_before_u() {
+        let _ = ffmpeg::init();
+        let source = [1, 2, 3, 4, 9, 8];
+        let yv12 = frame_from_bytes(&source, format(b"YV12", 2, 2, 2)).unwrap();
+        let yu12 = frame_from_bytes(&source, format(b"YU12", 2, 2, 2)).unwrap();
+        assert_eq!(active_rows(&yv12, 0, 2), vec![1, 2, 3, 4]);
+        assert_eq!(active_rows(&yv12, 1, 1), vec![8]);
+        assert_eq!(active_rows(&yv12, 2, 1), vec![9]);
+        assert_eq!(active_rows(&yu12, 1, 1), vec![9]);
+        assert_eq!(active_rows(&yu12, 2, 1), vec![8]);
+    }
+
+    #[test]
+    fn a_live_frame_keeps_its_pixels_when_the_next_one_is_filled() {
+        let _ = ffmpeg::init();
+        let first =
+            copy_packed(&[1, 2, 3, 4], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();
+        let second =
+            copy_packed(&[5, 6, 7, 8], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2).unwrap();
+        assert_eq!(active_rows(&first, 0, 4), vec![1, 2, 3, 4]);
+        assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);
+        drop(first);
+        let third = copy_packed(
+            &[9, 10, 11, 12],
+            format(b"YUYV", 2, 1, 4),
+            Pixel::YUYV422,
+            2,
+        )
+        .unwrap();
+        assert_eq!(active_rows(&third, 0, 4), vec![9, 10, 11, 12]);
+        assert_eq!(active_rows(&second, 0, 4), vec![5, 6, 7, 8]);
+    }
+
+    #[test]
+    fn short_and_odd_frames_keep_their_errors() {
+        let _ = ffmpeg::init();
+        let short = copy_packed(&[1], format(b"YUYV", 2, 1, 4), Pixel::YUYV422, 2);
+        assert!(matches!(short, Err(AsFFmpegError::InvalidFrameData { .. })));
+        let odd = frame_from_bytes(&[1, 2, 3, 4, 5, 6], format(b"YU12", 2, 2, 3));
+        assert!(matches!(odd, Err(AsFFmpegError::InvalidFrameStride { .. })));
+        let unsupported = frame_from_bytes(&[], format(b"XXXX", 1, 1, 1));
+        assert!(matches!(
+            unsupported,
+            Err(AsFFmpegError::UnsupportedFormat(_))
+        ));
+    }
+}

crates/camera/examples/cli.rs

@@ -26,9 +26,7 @@ fn main() {
         .unwrap();
 
     let _handle = selected_camera
-        .start_capturing(selected_format.0, |frame| {
-            drop(frame);
-        })
+        .start_capturing(selected_format.0, |_frame| {})
         .unwrap();
 
     std::thread::sleep(Duration::from_secs(5));

crates/camera/examples/diagnose.rs

@@ -196,6 +196,13 @@ fn capture_test(camera: &CameraInfo, mode: CaptureMode) {
     }
 }
 
+#[cfg(target_os = "linux")]
+fn describe_frame(frame: &cap_camera::CapturedFrame<'_>) -> String {
+    let _ = frame;
+    "<frame>".to_string()
+}
+
+#[cfg(not(target_os = "linux"))]
 fn describe_frame(frame: &cap_camera::CapturedFrame) -> String {
     #[cfg(target_os = "macos")]
     {