diff --git a/src-tauri/src/transcode/encoder.rs b/src-tauri/src/transcode/encoder.rs index 4ba8c76b8..5a905ead1 100644 --- a/src-tauri/src/transcode/encoder.rs +++ b/src-tauri/src/transcode/encoder.rs @@ -10,15 +10,23 @@ pub struct EncodedOutput { } pub trait H264Encoder: Send { + /// Submit one raw RGB frame for encoding. + /// + /// Returns the encoded outputs that became available as a result, in + /// submission order (0 or more). Asynchronous hardware encoders such as + /// VideoToolbox on Intel Macs buffer several frames internally, so a call + /// may return no output while an earlier frame is still in flight. Callers + /// must drain the tail with [`H264Encoder::finish`]. fn encode_frame( &mut self, rgb: &[u8], width: usize, height: usize, - ) -> Result; + ) -> Result, String>; - fn flush(&mut self) -> Result<(), String> { - Ok(()) + /// Flush the encoder and return every remaining buffered output, in order. + fn finish(&mut self) -> Result, String> { + Ok(Vec::new()) } fn sps(&self) -> &[u8]; @@ -56,7 +64,7 @@ impl H264Encoder for OpenH264Encoder { rgb: &[u8], width: usize, height: usize, - ) -> Result { + ) -> Result, String> { let rgb_slice = RgbSliceU8::new(rgb, (width, height)); let yuv = YUVBuffer::from_rgb_source(rgb_slice); @@ -67,11 +75,7 @@ impl H264Encoder for OpenH264Encoder { let raw = bs.to_vec(); if raw.is_empty() { - return Ok(EncodedOutput { - data: Vec::new(), - is_keyframe: false, - sample_size: 0, - }); + return Ok(Vec::new()); } let nal_units = split_annex_b(&raw); @@ -105,11 +109,11 @@ impl H264Encoder for OpenH264Encoder { sample_size += 4 + len; } - Ok(EncodedOutput { + Ok(vec![EncodedOutput { data, is_keyframe: is_key, sample_size, - }) + }]) } fn sps(&self) -> &[u8] { @@ -250,6 +254,30 @@ pub mod vt { } } } + + fn take_output(&mut self, frame: EncodedFrameData) -> EncodedOutput { + if !frame.sps.is_empty() { + self.sps = frame.sps; + } + if !frame.pps.is_empty() { + self.pps = frame.pps; + } + let sample_size = frame.data.len() as u32; + EncodedOutput { + data: frame.data, + is_keyframe: frame.is_keyframe, + sample_size, + } + } + + fn drain_ready(&mut self) -> Vec { + let mut outputs = Vec::new(); + while let Ok(frame) = self.rx.try_recv() { + let out = self.take_output(frame); + outputs.push(out); + } + outputs + } } impl H264Encoder for VideoToolboxEncoder { @@ -258,50 +286,36 @@ pub mod vt { rgb: &[u8], width: usize, height: usize, - ) -> Result { + ) -> Result, String> { self.rgb_to_i420(rgb, width, height); - let frame_data = FrameData::I420 { - y: &self.i420_y, - u: &self.i420_u, - v: &self.i420_v, - }; - - let opts = EncodeOptions::default(); let pts = self.next_pts; self.next_pts += 1; - self.inner - .encode(&frame_data, &opts, pts) - .map_err(|e| format!("VideoToolbox encode failed: {}", e))?; - - match self.rx.recv() { - Ok(frame) => { - if !frame.sps.is_empty() { - self.sps = frame.sps; - } - if !frame.pps.is_empty() { - self.pps = frame.pps; - } - let sample_size = frame.data.len() as u32; - Ok(EncodedOutput { - data: frame.data, - is_keyframe: frame.is_keyframe, - sample_size, - }) - } - Err(_) => Ok(EncodedOutput { - data: Vec::new(), - is_keyframe: false, - sample_size: 0, - }), + { + let frame_data = FrameData::I420 { + y: &self.i420_y, + u: &self.i420_u, + v: &self.i420_v, + }; + let opts = EncodeOptions::default(); + self.inner + .encode(&frame_data, &opts, pts) + .map_err(|e| format!("VideoToolbox encode failed: {}", e))?; } + + // Collect whatever the encoder has emitted so far without blocking. + // On Intel Macs the hardware encoder buffers several frames, so this + // may be empty until the pipeline fills; the tail is drained in + // finish(). Blocking here would deadlock on those buffered encoders. + Ok(self.drain_ready()) } - fn flush(&mut self) -> Result<(), String> { + fn finish(&mut self) -> Result, String> { self.inner .finish() - .map_err(|e| format!("VideoToolbox flush failed: {}", e)) + .map_err(|e| format!("VideoToolbox flush failed: {}", e))?; + Ok(self.drain_ready()) } fn sps(&self) -> &[u8] { diff --git a/src-tauri/src/transcode/transcode.rs b/src-tauri/src/transcode/transcode.rs index d3b406de7..859149995 100644 --- a/src-tauri/src/transcode/transcode.rs +++ b/src-tauri/src/transcode/transcode.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] -use crate::transcode::encoder::{create_encoder, H264Encoder}; +use crate::transcode::encoder::{create_encoder, EncodedOutput, H264Encoder}; use crate::transcode::parser::Parser; use crate::transcode::renderer::Renderer; use crate::transcode::{bitrate_for_resolution, compute_target_dimensions, OutputResolution}; @@ -898,6 +898,31 @@ fn encode_single_chunk( } } +fn append_sample( + nals: &mut Vec, + sample_sizes: &mut Vec, + keyframe_indices: &mut Vec, + sample_repeat_counts: &mut Vec, + submitted_repeats: &mut std::collections::VecDeque, + output: EncodedOutput, +) { + // Each output corresponds to one submitted frame in order. + let repeat_count = submitted_repeats.pop_front().unwrap_or(1); + if output.sample_size == 0 { + if let Some(last) = sample_repeat_counts.last_mut() { + *last += repeat_count; + } + return; + } + let sample_idx = sample_sizes.len(); + nals.extend_from_slice(&output.data); + sample_sizes.push(output.sample_size); + sample_repeat_counts.push(repeat_count); + if output.is_keyframe { + keyframe_indices.push(sample_idx); + } +} + fn encoder_thread_fn( rx: mpsc::Receiver, enc_w_fixed: Option, @@ -914,15 +939,20 @@ fn encoder_thread_fn( let mut sample_repeat_counts = Vec::new(); let mut max_enc_w: u32 = 0; let mut max_enc_h: u32 = 0; - let mut has_prev_sample = false; let mut sps = Vec::new(); let mut pps = Vec::new(); + // Repeat counts of frames submitted to the encoder whose encoded output may + // arrive on a later call (VideoToolbox on Intel Macs buffers frames). Output + // order matches submission order (frame reordering is disabled), so each + // emitted sample consumes the front entry. + let mut submitted_repeats: std::collections::VecDeque = std::collections::VecDeque::new(); + // Trailing duplicate frames that extend the last emitted sample's duration. + let mut trailing_repeat: u32 = 0; + for prepared in rx { if prepared.rgb.is_empty() { - if has_prev_sample { - *sample_repeat_counts.last_mut().unwrap() += prepared.repeat_count; - } + trailing_repeat += prepared.repeat_count; continue; } @@ -935,34 +965,62 @@ fn encoder_thread_fn( )?); } - let enc = encoder.as_mut().unwrap(); - let output = enc.encode_frame(&prepared.rgb, prepared.width, prepared.height)?; - - if output.sample_size > 0 { - let sample_idx = sample_sizes.len(); - nals.extend_from_slice(&output.data); - sample_sizes.push(output.sample_size); - sample_repeat_counts.push(prepared.repeat_count); - if output.is_keyframe { - keyframe_indices.push(sample_idx); - } - max_enc_w = max_enc_w.max(prepared.width as u32); - max_enc_h = max_enc_h.max(prepared.height as u32); + max_enc_w = max_enc_w.max(prepared.width as u32); + max_enc_h = max_enc_h.max(prepared.height as u32); + submitted_repeats.push_back(prepared.repeat_count); - if !enc.sps().is_empty() { - sps = enc.sps().to_vec(); - } - if !enc.pps().is_empty() { - pps = enc.pps().to_vec(); - } - has_prev_sample = true; - } else if has_prev_sample { - *sample_repeat_counts.last_mut().unwrap() += prepared.repeat_count; + let enc = encoder.as_mut().unwrap(); + let outputs = enc.encode_frame(&prepared.rgb, prepared.width, prepared.height)?; + for output in outputs { + append_sample( + &mut nals, + &mut sample_sizes, + &mut keyframe_indices, + &mut sample_repeat_counts, + &mut submitted_repeats, + output, + ); + } + if !enc.sps().is_empty() { + sps = enc.sps().to_vec(); + } + if !enc.pps().is_empty() { + pps = enc.pps().to_vec(); } } if let Some(ref mut enc) = encoder { - let _ = enc.flush(); + let outputs = enc.finish()?; + for output in outputs { + append_sample( + &mut nals, + &mut sample_sizes, + &mut keyframe_indices, + &mut sample_repeat_counts, + &mut submitted_repeats, + output, + ); + } + if !enc.sps().is_empty() { + sps = enc.sps().to_vec(); + } + if !enc.pps().is_empty() { + pps = enc.pps().to_vec(); + } + } + + // Frames that never produced their own sample (e.g. a rare empty bitstream) + // fold into the previous sample's duration to preserve overall timing. + let leftover: u32 = submitted_repeats.iter().sum(); + if leftover > 0 { + if let Some(last) = sample_repeat_counts.last_mut() { + *last += leftover; + } + } + if trailing_repeat > 0 { + if let Some(last) = sample_repeat_counts.last_mut() { + *last += trailing_repeat; + } } let (out_sps, out_pps) = if canonical_sps.is_empty() {