From aa99039daaaa7ba4de9402889a9a4bbe8485aa6d Mon Sep 17 00:00:00 2001 From: unRekable Date: Tue, 10 Nov 2026 20:56:47 +0100 Subject: [PATCH 1/7] Encode a share or camera with the machine's own hardware encoder first Windows Media Foundation lists the hardware H.264 encoders, and Coded now opens the first one that takes NV12 and answers H.264: async MFT fed NV12 at a promised stride, drained through its event queue, CBR at the asked bitrate, keyframes through CODECAPI_AVEncVideoForceKeyFrame, output normalized to Annex B. A machine with no hardware MFT, or one that refuses its setup, keeps the OpenH264 path and its words, and skips the download only when the hardware will take the job. The OpenH264 retry, cache and failure words are untouched. --- crates/mediad/Cargo.toml | 5 +- crates/mediad/src/lib.rs | 3 + crates/mediad/src/mf_encoder.rs | 610 +++++++++++++++++++++++++++ crates/mediad/src/picture/desktop.rs | 154 ++++--- docs/product/06-calls.md | 2 +- 5 files changed, 723 insertions(+), 51 deletions(-) create mode 100644 crates/mediad/src/mf_encoder.rs diff --git a/crates/mediad/Cargo.toml b/crates/mediad/Cargo.toml index 4090f61f..d9ea6664 100644 --- a/crates/mediad/Cargo.toml +++ b/crates/mediad/Cargo.toml @@ -91,8 +91,9 @@ webpki-roots = "1" # Already in the tree through cpal and mio. windows-sys = { version = "0.61", features = ["Win32_Media", "Win32_System_Threading", "Win32_Foundation", "Win32_Graphics_Gdi", "Win32_Graphics_Dwm", "Win32_Storage_Xps", "Win32_UI_HiDpi", "Win32_UI_WindowsAndMessaging"] } # A screen share through DXGI Desktop Duplication (`picture/duplicate`): Direct3D 11 and DXGI are -# COM, which the `windows` crate wraps. The version notifier already builds. -windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi"] } +# COM, which the `windows` crate wraps. The version notifier already builds. Media Foundation's +# hardware H.264 encoder (`mf_encoder`) is COM too, with its variants (`Variant`) beside it. +windows = { version = "0.62", features = ["Win32_Foundation", "Win32_Graphics_Direct3D", "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", "Win32_Graphics_Gdi", "Win32_Media_MediaFoundation", "Win32_System_Com", "Win32_System_Ole", "Win32_System_Variant"] } [target.'cfg(unix)'.dependencies] # The priority of video's threads. Already in the tree through mio and cpal. diff --git a/crates/mediad/src/lib.rs b/crates/mediad/src/lib.rs index 8ccc3c0f..27af92dc 100644 --- a/crates/mediad/src/lib.rs +++ b/crates/mediad/src/lib.rs @@ -37,6 +37,9 @@ pub mod hear; pub mod inproc; pub mod jitter; pub mod media; +/// The machine's own H.264 hardware encoder, ahead of the OpenH264 download. Windows only. +#[cfg(windows)] +mod mf_encoder; pub mod ogg; pub mod opus; pub mod picture; diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs new file mode 100644 index 00000000..8004ee38 --- /dev/null +++ b/crates/mediad/src/mf_encoder.rs @@ -0,0 +1,610 @@ +//! The Windows Media Foundation H.264 encoder, the machine's own hardware ahead of the +//! OpenH264 download (`picture/desktop`), which stays as the fallback. +//! +//! A hardware encoder is an asynchronous Media Foundation transform (MFT): it is found with +//! `MFTEnumEx` over the registered hardware encoders, woken by `MF_TRANSFORM_ASYNC_UNLOCK`, +//! fed NV12 frames through `ProcessInput`, and drained through its event queue +//! (`METransformNeedInput`, `METransformHaveOutput`) with `ProcessOutput`. The rate is CBR +//! at the bitrate the track asked for (`MF_MT_AVG_BITRATE` and +//! `CODECAPI_AVEncCommonRateControlMode`), a keyframe is asked for with +//! `CODECAPI_AVEncVideoForceKeyFrame`, and the output is normalized to the Annex B byte +//! stream the SFU and the OpenH264 decoder take. A frame whose bits are not out yet comes +//! back as an empty [`Encoded`], which the share already treats as a skip; they leave on a +//! later frame's pump. +//! +//! Everything here runs where the encoder was opened: the share's or camera's capture +//! thread, COM in the Multi Threaded Apartment, Media Foundation started without its +//! sockets. `Outgoing` only ever moves between threads before an encoder exists, and MTA +//! objects are callable from every thread of the process, so the `Send` on [`Encoder`] is +//! honest. What the encoder keeps between frames is its bitrate, its frame rate, and +//! whether it has an input slot to fill; a change of frame size is not a tune but a new +//! encoder (`coded` in `desktop`), as with OpenH264. + +use std::ptr::null_mut; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; +use windows::core::Interface; +use windows::Win32::Foundation::{RPC_E_CHANGED_MODE, VARIANT_BOOL}; +use windows::Win32::Media::MediaFoundation::{ + CODECAPI_AVEncCommonLowLatency, CODECAPI_AVEncCommonMeanBitRate, CODECAPI_AVEncCommonRateControlMode, + CODECAPI_AVEncMPVDefaultBPictureCount, CODECAPI_AVEncVideoForceKeyFrame, CODECAPI_AVLowLatencyMode, + IMFActivate, IMFCodecAPI, IMFMediaEventGenerator, IMFTransform, MFVideoInterlace_Progressive, + MF_E_NO_EVENTS_AVAILABLE, MF_EVENT_FLAG_NO_WAIT, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, + MF_MT_DEFAULT_STRIDE, MF_MT_FIXED_SIZE_SAMPLES, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, + MF_MT_MAJOR_TYPE, MF_MT_MPEG2_LEVEL, MF_MT_MPEG2_PROFILE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SAMPLE_SIZE, + MF_MT_SUBTYPE, MF_TRANSFORM_ASYNC_UNLOCK, MF_VERSION, METransformHaveOutput, METransformNeedInput, + MFTEnumEx, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, MFT_ENUM_FLAG_SORTANDFILTER, + MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_STREAMING, + MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER, MFT_REGISTER_TYPE_INFO, MFCreateMediaType, + MFCreateMemoryBuffer, MFCreateSample, MFMediaType_Video, MFStartup, MFSTARTUP_NOSOCKET, + MFVideoFormat_H264, MFVideoFormat_NV12, MFSampleExtension_CleanPoint, eAVEncCommonRateControlMode_CBR, + eAVEncH264VLevel3_1, eAVEncH264VLevel5_1, eAVEncH264VProfile_ConstrainedBase, +}; +use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoTaskMemFree}; +use windows::Win32::System::Variant::{VT_BOOL, VT_UI4, VARIANT, VariantInit}; + +use crate::picture::{Ask, Encoded, Planes}; + +/// How long one frame waits for the machine's encoder to answer. Hardware encoders in low +/// latency answer inside a frame's time; when one does not, the frame goes out empty and +/// its bits leave on the next pump, the way a skip already does. +const ANSWER_WAIT: Duration = Duration::from_millis(120); +/// How often the event queue is looked at while waiting for that answer. +const POLL: Duration = Duration::from_millis(1); +/// The NV12 stride promised to the encoder, the width rounded up as the hardware wants it. +const STRIDE_ALIGN: usize = 16; + +/// Which backend a track's encoder takes: the machine's hardware on a clean probe, the +/// OpenH264 download on any failure. The one place that decision lives, so tests can hold +/// it against a probe result without hardware or the network. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Choice { + MediaFoundation, + OpenH264, +} + +/// The backend a probe result means. +pub fn choose(probe: &Result<(), String>) -> Choice { + if probe.is_ok() { Choice::MediaFoundation } else { Choice::OpenH264 } +} + +/// Whether this machine has a hardware H.264 encoder Media Foundation can drive, asked once +/// and remembered for the process: the share and camera paths read this to keep the +/// OpenH264 download off a machine that will never need it, and [`Encoder::open`] asks the +/// system again before it opens. +pub fn available() -> Result<(), String> { + static AVAILABLE: OnceLock> = OnceLock::new(); + AVAILABLE.get_or_init(probe).clone() +} + +/// One look at what the machine lists, without opening anything. Fails in its own words: a +/// machine with no hardware MFT shares through OpenH264, and so does one whose COM or Media +/// Foundation would not start. +fn probe() -> Result<(), String> { + platform()?; + if enumerate() == 0 { + return Err("This machine lists no hardware H.264 encoder.".to_owned()); + } + Ok(()) +} + +/// How many hardware H.264 encoders this machine lists, taking NV12 and answering H.264. +fn enumerate() -> u32 { + let takes = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_NV12 }; + let answers = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_H264 }; + let mut list: *mut Option = null_mut(); + let mut count = 0u32; + // SAFETY: both registration types outlive the call, and the list it hands back is freed + // whole below, after the count is read. Nothing here keeps a handle into it. + let _ = unsafe { + MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, Some(&takes), Some(&answers), &mut list, &mut count) + }; + // SAFETY: the list is MFTEnumEx's own allocation, freed whole; no element is read. + unsafe { CoTaskMemFree(Some(list.cast::())) }; + count +} + +/// COM in the Multi Threaded Apartment, per thread, and Media Foundation, per process. +/// Both keep count, so an encoder thread that meets another's work answers at once. +fn platform() -> Result<(), String> { + // SAFETY: the apartment call is the documented per-thread init; its result is checked. + let started = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; + if started.is_err() { + return Err(if started == RPC_E_CHANGED_MODE { + "This thread already holds a COM apartment the encoder cannot share.".to_owned() + } else { + format!("COM would not start for the encoder: {started}") + }); + } + // The init is never undone: the capture thread outlives the encoder by a frame at most, + // and Media Foundation's own teardown races nothing that outlives it here. + // SAFETY: the version and the no-sockets flag are the constants Media Foundation reads. + unsafe { MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET) }.map_err(|err| format!("Media Foundation would not start: {err}")) +} + +/// The machine's hardware H.264 encoder, held as the async MFT, its event queue, and the +/// codec properties that survive a frame. +pub struct Encoder { + transform: IMFTransform, + events: IMFMediaEventGenerator, + /// `None` when the MFT exposes no codec properties, which leaves a keyframe ask and a + /// bitrate move without their handle; the picture still encodes. + codec: Option, + width: usize, + height: usize, + /// The stride promised in the input type: the width rounded up as the hardware wants. + stride: usize, + /// One NV12 frame at that stride: the Y plane and the two interleaved chroma planes. + frame: usize, + /// What the encoder holds now, so a tune is only sent when the ask moved. + bps: u32, + fps: f32, + /// Whether the encoder has an input slot open: a `METransformNeedInput` seen and not + /// yet fed. + wants_input: bool, +} + +/// SAFETY: the encoder is opened, driven, and dropped on one thread (the share's or the +/// camera's capture thread); `Outgoing` moves only before any encoder exists. The COM +/// apartment is the MTA and Media Foundation objects answer from every thread of the +/// process, so even a move after the fact would stay within what the platform allows. +unsafe impl Send for Encoder {} + +impl Encoder { + /// The first hardware encoder that takes NV12 and answers H.264, set up for CBR at + /// `bps` and at most `fps`, level 5.1 for a screen and 3.1 for a camera, at the profile + /// the SFU takes. Its own words when there is no such encoder or one would not take its + /// setup; the caller falls back to OpenH264 on any of them. + pub fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + platform()?; + let (width, height) = (width.max(2) as usize, height.max(2) as usize); + let stride = (width + STRIDE_ALIGN - 1) & !(STRIDE_ALIGN - 1); + let frame = stride * height * 3 / 2; + + let mut list: *mut Option = null_mut(); + let mut count = 0u32; + let takes = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_NV12 }; + let answers = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_H264 }; + // SAFETY: as in `enumerate`, the registration types outlive the call and the list is + // freed below, after its first entry has been taken out of it. + unsafe { MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, Some(&takes), Some(&answers), &mut list, &mut count) } + .map_err(|err| format!("the hardware encoders could not be listed: {err}"))?; + let first = if count > 0 { + // SAFETY: `count` says the list holds at least one entry, and reading it moves + // the interface out of MFTEnumEx's allocation before that allocation is freed. + Some(unsafe { std::ptr::read(list.add(0)) }) + } else { + None + }; + // SAFETY: as above, the list is freed whole. + unsafe { CoTaskMemFree(Some(list.cast::())) }; + let Some(activate) = first.flatten() else { + return Err("This machine lists no hardware H.264 encoder.".to_owned()); + }; + // SAFETY: the activation object hands out the transform it stands for, once. + let transform: IMFTransform = unsafe { activate.ActivateObject() } + .map_err(|err| format!("the hardware encoder could not be started: {err}"))?; + + // SAFETY: from here on every call is on interfaces this open creates or this + // encoder holds, whose pointers the calls only read. + unsafe { + // A hardware encoder answers on its own clock. The unlock tells Media + // Foundation this client will drive it through its event queue, before any + // type is set. + let attributes = transform.GetAttributes().map_err(|err| format!("the hardware encoder told nothing about itself: {err}"))?; + attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1).map_err(|err| format!("the hardware encoder could not be unlocked: {err}"))?; + + let input = input_type(width, height, stride, frame, fps)?; + transform.SetInputType(0, &input, 0).map_err(|err| format!("the hardware encoder would not take NV12: {err}"))?; + + match output_type(width, height, bps, fps, screen, true) { + Ok(output) => { + if transform.SetOutputType(0, &output, 0).is_err() { + // A profile or level this encoder does not keep is its own to pick: + // ask again without them, and say so once. + let plain = output_type(width, height, bps, fps, screen, false)?; + transform.SetOutputType(0, &plain, 0) + .map_err(|err| format!("the hardware encoder would not take an H.264 output: {err}"))?; + tracing::debug!("mediad: the hardware encoder chose its own H.264 profile."); + } + } + Err(why) => return Err(why), + } + + let codec: Option = transform.cast().ok(); + if let Some(codec) = &codec { + // Both rates are said twice on purpose: once on the output type, which the + // MFT reads as it opens, and once through the codec properties, which the + // rate control holds for the frames after. A refusal is logged and lived + // with: the encoder still answers, at a rate of its own choosing. + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonRateControlMode, &counted(eAVEncCommonRateControlMode_CBR.0 as u32)) { + tracing::debug!("mediad: the hardware encoder took no CBR rate control: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(bps)) { + tracing::debug!("mediad: the hardware encoder took no mean bitrate: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonLowLatency, &yes()) { + tracing::debug!("mediad: the hardware encoder took no low latency switch: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVLowLatencyMode, &yes()) { + tracing::debug!("mediad: the hardware encoder took no low latency mode: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncMPVDefaultBPictureCount, &counted(0)) { + tracing::debug!("mediad: the hardware encoder took no B picture count: {err}"); + } + } + + let events: IMFMediaEventGenerator = transform.cast().map_err(|err| format!("the hardware encoder has no event queue: {err}"))?; + transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0).map_err(|err| format!("the hardware encoder would not start streaming: {err}"))?; + transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0).map_err(|err| format!("the hardware encoder would not start streaming: {err}"))?; + + let mut encoder = Encoder { transform, events, codec, width, height, stride, frame, bps, fps, wants_input: false }; + // The first input slot is usually open before the first frame; wait a moment + // for it, so the first frame of a share is fed at once and not dropped. + let until = Instant::now() + ANSWER_WAIT; + while !encoder.wants_input && Instant::now() < until { + encoder.pump(&mut Vec::new(), &mut false)?; + if !encoder.wants_input { + std::thread::sleep(POLL); + } + } + Ok(encoder) + } + } + + /// One frame in, whatever is out. A frame whose bits are still on their way comes back + /// empty, like a skip; an encoded frame comes back marked when it is a keyframe, so + /// the share sends it as one. + pub fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { + if planes.width != self.width || planes.height != self.height { + return Err(format!("The encoder was opened for {}×{} and handed {}×{}.", self.width, self.height, planes.width, planes.height)); + } + self.tune(ask); + if ask.key { + self.request_key(); + } + // What is already out, before this frame goes in: the machine may be a frame ahead. + let mut bytes = Vec::new(); + let mut key = false; + self.pump(&mut bytes, &mut key)?; + if self.wants_input { + self.submit(planes, ask)?; + // A bounded moment for this frame's own bits. When they are not out in time, + // the empty frame goes and the pump catches them next time. + let until = Instant::now() + ANSWER_WAIT; + while bytes.is_empty() && Instant::now() < until { + self.pump(&mut bytes, &mut key)?; + if bytes.is_empty() { + std::thread::sleep(POLL); + } + } + } + if bytes.is_empty() { + key = false; + } + Ok(Encoded { bytes, key }) + } + + /// The rate asked for, if it moved. The frame rate is set at open and is not moved + /// again; a moving ask for it is a no-op here. + fn tune(&mut self, ask: &Ask) { + if ask.bps == self.bps || ask.bps == 0 { + return; + } + let Some(codec) = &self.codec else { + return; + }; + // SAFETY: the property is read through the codec handle this encoder holds, and + // the variant outlives the call. + match unsafe { codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(ask.bps)) } { + Ok(()) => self.bps = ask.bps, + Err(err) => tracing::warn!("mediad: the hardware encoder refused a bitrate of {}: {err}", ask.bps), + } + } + + /// A keyframe for the next frame in. The property carries no value; setting it is the + /// ask. An encoder without codec properties cannot be asked, and keeps its own + /// keyframe pace. + fn request_key(&mut self) { + let Some(codec) = &self.codec else { + return; + }; + // SAFETY: as in `tune`, with a property whose value is ignored. + if let Err(err) = unsafe { codec.SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &counted(0)) } { + tracing::debug!("mediad: the hardware encoder refused a keyframe: {err}"); + } + } + + /// This frame as a sample, its planes packed NV12 at the promised stride. + fn submit(&mut self, planes: &Planes, ask: &Ask) -> Result<(), String> { + // SAFETY: the sample is created here, filled here, and handed to the transform, + // which owns its reference from `ProcessInput` on; the buffer pointer is used + // within its lock below and gone before the lock is let go. + unsafe { + let buffer = MFCreateMemoryBuffer(self.frame as u32).map_err(|err| format!("the frame could not be given room: {err}"))?; + let mut data: *mut u8 = null_mut(); + buffer.Lock(&mut data, None, None).map_err(|err| format!("the frame could not be written: {err}"))?; + nv12(planes, self.stride, std::slice::from_raw_parts_mut(data, self.frame)); + let _ = buffer.SetCurrentLength(self.frame as u32); + buffer.Unlock().map_err(|err| format!("the frame could not be handed over: {err}"))?; + + let sample = MFCreateSample().map_err(|err| format!("the frame could not be wrapped: {err}"))?; + sample.AddBuffer(&buffer).map_err(|err| format!("the frame could not be wrapped: {err}"))?; + // The ask's clock is milliseconds; Media Foundation's is 100 nanoseconds. + sample.SetSampleTime((ask.at.as_millis() as i64).saturating_mul(10_000)).map_err(|err| format!("the frame's time was refused: {err}"))?; + let lasting = (10_000_000.0 / self.fps.max(1.0)) as i64; + sample.SetSampleDuration(lasting.max(1)).map_err(|err| format!("the frame's length was refused: {err}"))?; + self.transform.ProcessInput(0, &sample, 0).map_err(|err| format!("the hardware encoder would not take the frame: {err}"))?; + self.wants_input = false; + Ok(()) + } + } + + /// Everything the event queue holds now, without waiting: frames out, and the open + /// input slot remembered for the next frame in. + fn pump(&mut self, bytes: &mut Vec, key: &mut bool) -> Result<(), String> { + loop { + // SAFETY: the event generator belongs to this encoder and is not held + // elsewhere; each event is this call's own, read once and dropped. + let event = match unsafe { self.events.GetEvent(MF_EVENT_FLAG_NO_WAIT) } { + Ok(event) => event, + Err(err) if err.code() == MF_E_NO_EVENTS_AVAILABLE => return Ok(()), + Err(err) => return Err(format!("the hardware encoder's events failed: {err}")), + }; + let kind = unsafe { event.GetType() }.map_err(|err| format!("the hardware encoder's events could not be read: {err}"))?; + if kind == METransformHaveOutput.0 as u32 { + self.take_output(bytes, key)?; + } else if kind == METransformNeedInput.0 as u32 { + self.wants_input = true; + } + // METransformDrainComplete and METransformInputStreamStateChanged need nothing + // from a share: nothing is ever drained, and the input state is read back + // through `wants_input` alone. + } + } + + /// One encoded frame out of the transform, its bytes appended and its keyframe-ness + /// remembered. + fn take_output(&mut self, bytes: &mut Vec, key: &mut bool) -> Result<(), String> { + // SAFETY: the output slot belongs to this call; the sample the transform fills is + // taken out of it and read before it is dropped here. + unsafe { + let mut slot = [MFT_OUTPUT_DATA_BUFFER { + dwStreamID: 0, + pSample: std::mem::ManuallyDrop::new(None), + dwStatus: 0, + pEvents: std::mem::ManuallyDrop::new(None), + }]; + let mut status = 0u32; + self.transform.ProcessOutput(0, &mut slot, &mut status).map_err(|err| format!("the hardware encoder gave no frame: {err}"))?; + let Some(sample) = std::mem::ManuallyDrop::take(&mut slot[0].pSample) else { + return Ok(()); + }; + if sample.GetUINT32(&MFSampleExtension_CleanPoint).is_ok() { + *key = true; + } + // The sample's buffers are made one here, whatever count the encoder used. + let contiguous = sample.ConvertToContiguousBuffer().map_err(|err| format!("the encoded frame could not be read: {err}"))?; + let mut data: *mut u8 = null_mut(); + let mut length = 0u32; + contiguous.Lock(&mut data, None, Some(&mut length)).map_err(|err| format!("the encoded frame could not be read: {err}"))?; + bytes.extend_from_slice(&annex_b(std::slice::from_raw_parts(data, length as usize))); + contiguous.Unlock().map_err(|err| format!("the encoded frame could not be let go: {err}"))?; + Ok(()) + } + } +} + +impl Drop for Encoder { + fn drop(&mut self) { + // SAFETY: the transform is still owned here; both messages are its own. + unsafe { + let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + let _ = self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + } + } +} + +/// The input type the encoder is fed: NV12 at the promised stride, whole frames, at the +/// size and rate the track asked for. +fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result { + // SAFETY: the media type is created here and returned; the caller sets it before drop. + unsafe { + let kind = MFCreateMediaType().map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_NV12).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT64(&MF_MT_FRAME_SIZE, size(width, height)).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT64(&MF_MT_FRAME_RATE, rate(fps)).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_ALL_SAMPLES_INDEPENDENT, 1).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_FIXED_SIZE_SAMPLES, 1).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_SAMPLE_SIZE, frame as u32).map_err(|err| format!("the input type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_DEFAULT_STRIDE, stride as u32).map_err(|err| format!("the input type could not be built: {err}"))?; + Ok(kind) + } +} + +/// The output type asked for: H.264 at the bitrate, size and rate of the track, at the +/// profile the SFU takes when `with_profile` says so. An encoder that keeps none of them +/// is asked again without. +fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool, with_profile: bool) -> Result { + let level = if screen { eAVEncH264VLevel5_1 } else { eAVEncH264VLevel3_1 }; + // SAFETY: as in `input_type`, the type is returned to be set. + unsafe { + let kind = MFCreateMediaType().map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetGUID(&MF_MT_MAJOR_TYPE, &MFMediaType_Video).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetGUID(&MF_MT_SUBTYPE, &MFVideoFormat_H264).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_AVG_BITRATE, bps).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT64(&MF_MT_FRAME_SIZE, size(width, height)).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT64(&MF_MT_FRAME_RATE, rate(fps)).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, size(1, 1)).map_err(|err| format!("the output type could not be built: {err}"))?; + if with_profile { + // Constrained baseline is what the SFU takes, and what OpenH264 sends; a level + // holds the size and rate, 5.1 for a 1440p share at 60. + kind.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_ConstrainedBase.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_MPEG2_LEVEL, level.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; + } + Ok(kind) + } +} + +/// A frame size as Media Foundation packs it: width high, height low. +fn size(width: usize, height: usize) -> u64 { + ((width as u64) << 32) | height as u64 +} + +/// A frame rate as Media Foundation packs it: thousandths of a frame a second over 1000. +fn rate(fps: f32) -> u64 { + ((fps * 1000.).round().max(1.) as u64) << 32 | 1000 +} + +/// A `VARIANT` holding a count, which is how the codec properties are read. +fn counted(count: u32) -> VARIANT { + // SAFETY: a VARIANT is initialized here and read by the property call it is handed to. + unsafe { + let mut value = VariantInit(); + value.Anonymous.Anonymous.vt = VT_UI4; + value.Anonymous.Anonymous.Anonymous.ulVal = count; + value + } +} + +/// A `VARIANT` holding TRUE, for the switches. +fn yes() -> VARIANT { + // SAFETY: as in `counted`, with a boolean. + unsafe { + let mut value = VariantInit(); + value.Anonymous.Anonymous.vt = VT_BOOL; + value.Anonymous.Anonymous.Anonymous.boolVal = VARIANT_BOOL(-1); + value + } +} + +/// `planes` as NV12 into `out`, Y rows at `stride` and the two chroma planes interleaved +/// under them. The padding rows and bytes stay what the buffer was made with, and a buffer +/// too short takes what fits and nothing past it. +fn nv12(planes: &Planes, stride: usize, out: &mut [u8]) { + let (width, height) = (planes.width, planes.height); + for row in 0..height { + let at = row * stride; + if at + width > out.len() { + return; + } + out[at..at + width].copy_from_slice(&planes.y[row * width..(row + 1) * width]); + } + let chroma = width / 2; + for row in 0..height / 2 { + let at = height * stride + row * stride; + for col in 0..chroma { + let cell = at + col * 2; + if cell + 1 >= out.len() { + return; + } + let from = row * chroma + col; + out[cell] = planes.u[from]; + out[cell + 1] = planes.v[from]; + } + } +} + +/// The Annex B byte stream the SFU and the decoder take: every NAL unit behind a start +/// code. Media Foundation's H.264 encoder answers in Annex B already; an encoder that +/// answers in length-prefixed units instead is turned here, so both backends send the same +/// shape. A stream that is Annex B already is not touched, and a length the reading cannot +/// finish leaves the rest out rather than running past the bytes there are. +fn annex_b(bytes: &[u8]) -> Vec { + if bytes.starts_with(&[0, 0, 1]) || bytes.starts_with(&[0, 0, 0, 1]) { + return bytes.to_vec(); + } + let mut out = Vec::with_capacity(bytes.len() + 12); + let mut at = 0usize; + while at + 4 <= bytes.len() { + let length = u32::from_be_bytes([bytes[at], bytes[at + 1], bytes[at + 2], bytes[at + 3]]) as usize; + at += 4; + if length == 0 || at + length > bytes.len() { + break; + } + out.extend_from_slice(&[0, 0, 0, 1]); + out.extend_from_slice(&bytes[at..at + length]); + at += length; + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_clean_probe_means_hardware_and_any_failure_means_the_download() { + assert_eq!(choose(&Ok(())), Choice::MediaFoundation); + assert_eq!(choose(&Err("This machine lists no hardware H.264 encoder.".to_owned())), Choice::OpenH264); + assert_eq!(choose(&Err(String::new())), Choice::OpenH264); + } + + /// A share's own 4×4 picture, packed the way the encoder is promised its stride. + #[test] + fn nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved() { + let rgba = vec![255u8; 4 * 4 * 4]; + let planes = crate::picture::i420(&rgba, 4, 4, 4, 4, false); + let stride = 16; + let mut out = vec![7u8; 4 * 16 * 3 / 2]; + nv12(&planes, stride, &mut out); + // Four Y rows at the stride, the padding between and after them untouched. + for row in 0..4 { + let at = row * stride; + assert_eq!(out[at..at + 4], planes.y[row * 4..(row + 1) * 4], "row {row}"); + assert!(out[at + 4..at + stride].iter().all(|&byte| byte == 7), "padding of row {row} was written"); + } + // Chroma under the luma, one U and one V per pair of pixels. + for row in 0..2 { + let at = 4 * stride + row * stride; + for col in 0..2 { + assert_eq!(out[at + col * 2], planes.u[row * 2 + col], "u at {row},{col}"); + assert_eq!(out[at + col * 2 + 1], planes.v[row * 2 + col], "v at {row},{col}"); + } + } + } + + /// A short output writes what it can and stops, rather than running past the buffer. + #[test] + fn nv12_of_a_short_buffer_writes_nothing_past_it() { + let rgba = vec![128u8; 4 * 4 * 4]; + let planes = crate::picture::i420(&rgba, 4, 4, 4, 4, false); + let mut out = vec![0u8; 4]; // holds one Y row, no chroma + nv12(&planes, 4, &mut out); + assert_eq!(out, planes.y[..4]); + } + + #[test] + fn annex_b_already_annex_b_is_untouched() { + let stream = [0u8, 0, 0, 1, 0x65, 1, 2, 3, 0, 0, 0, 1, 0x41, 4, 5]; + assert_eq!(annex_b(&stream), stream.to_vec()); + let three = [0u8, 0, 1, 0x65, 1, 2, 3]; + assert_eq!(annex_b(&three), three.to_vec()); + } + + /// The other shape an H.264 byte stream takes: four byte lengths, one per unit. + #[test] + fn length_prefixed_units_come_out_behind_start_codes() { + let stream = [0u8, 0, 0, 4, 0x65, 1, 2, 3, 0, 0, 0, 2, 0x41, 4]; + assert_eq!(annex_b(&stream), vec![0, 0, 0, 1, 0x65, 1, 2, 3, 0, 0, 0, 1, 0x41, 4]); + } + + #[test] + fn a_truncated_or_empty_stream_never_runs_past_itself() { + assert!(annex_b(&[]).is_empty()); + // The length says more than there is: nothing comes out, nothing runs past. + assert!(annex_b(&[0, 0, 0, 99, 0x65]).is_empty()); + assert_eq!(annex_b(&[0, 0, 0, 2, 0x41, 4, 9, 9]), vec![0, 0, 0, 1, 0x41, 4]); + } + + #[test] + fn sizes_and_rates_are_packed_width_over_height_and_thousandths() { + assert_eq!(size(2560, 1440), (2560u64 << 32) | 1440); + assert_eq!(rate(60.), (60_000u64 << 32) | 1000); + assert_eq!(rate(29.97), (29_970u64 << 32) | 1000); + assert_eq!(rate(0.), 1u64 << 32 | 1000); + } +} diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index e0260fad..5d4bed6a 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -1,9 +1,13 @@ -//! The desktop's pictures: OpenH264 (Cisco's binary, fetched at run time), nokhwa's camera -//! (Media Foundation, AVFoundation, V4L2), and the screen through GDI, Core Graphics, the X11 +//! The desktop's pictures: a share or camera encoded first by the machine's own hardware +//! through Media Foundation (`mf_encoder`, Windows), else by OpenH264 (Cisco's binary, +//! fetched at run time), which decodes whatever arrives either way. nokhwa's camera (Media +//! Foundation, AVFoundation, V4L2), and the screen through GDI, Core Graphics, the X11 //! root, or the Linux desktop portal. Moved here from `picture.rs`, which keeps what phones share. use super::*; use crate::rt; +#[cfg(windows)] +use crate::mf_encoder; use openh264::decoder::{Decoder, DecoderConfig}; use openh264::encoder::{BitRate, Complexity, Encoder, EncoderConfig, FrameRate, FrameType, Level, Profile, RateControlMode, UsageType}; use openh264::formats::YUVSource; @@ -41,21 +45,29 @@ impl YUVSource for Planes { } struct Coded { - encoder: Encoder, + backend: Backend, width: u32, height: u32, - /// What the encoder holds now. - bps: u32, - fps: f32, - /// OpenH264 sets up on the first frame, and takes no options before that. - begun: bool, } -impl Coded { - /// `bps` is also the most this encoder ever spends: [`Coded::tune`] only goes under it, and - /// so is `fps`. A share's encoder is level 5.1, which holds 1440p at 60, and spreads its work - /// over threads: one thread could not encode 1080p inside a frame at 60. - fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { +/// One of the two encoders behind [`Coded`]: the machine's own hardware, or Cisco's +/// OpenH264 when that could not be set up. +enum Backend { + /// Windows Media Foundation's hardware encoders (NVIDIA, Intel, AMD), which take no + /// download and hardly any CPU. + #[cfg(windows)] + MediaFoundation(mf_encoder::Encoder), + /// The downloaded Cisco binary. It sets up on its first frame, and takes no options + /// before that. + OpenH264 { encoder: Encoder, begun: bool, bps: u32, fps: f32 }, +} + +impl Backend { + /// The OpenH264 fallback, opened the way it always was: `bps` is also the most this + /// encoder ever spends: [`Backend::encode`] only tunes under it, and so is `fps`. A + /// share's encoder is level 5.1, which holds 1440p at 60, and spreads its work over + /// threads: one thread could not encode 1080p inside a frame at 60. + fn open_h264(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { let api = load_api()?; let threads = if screen { std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 8) as u16 } else { 1 }; let config = EncoderConfig::new() @@ -70,47 +82,92 @@ impl Coded { .complexity(if screen && fps > 30. { Complexity::Low } else { Complexity::Medium }) .num_threads(threads); let encoder = Encoder::with_api_config(api, config).map_err(|err| err.to_string())?; - Ok(Coded { encoder, width, height, bps, fps, begun: false }) + Ok(Backend::OpenH264 { encoder, begun: false, bps, fps }) } - /// The rate and frame rate asked for, if they moved and the encoder can take them. - fn tune(&mut self, ask: &Ask) { - if !self.begun { - return; - } - if ask.bps != self.bps && ask.bps > 0 { - let mut info = SBitrateInfo { iLayer: SPATIAL_LAYER_ALL, iBitrate: ask.bps as i32 }; - // SAFETY: OpenH264 reads this option as an `SBitrateInfo`, which outlives the call, - // and copies it. The encoder set up on its first frame. - let refused = unsafe { self.encoder.raw_api().set_option(ENCODER_OPTION_BITRATE, (&raw mut info).cast()) }; - if refused == 0 { - self.bps = ask.bps; - } else { - tracing::warn!("mediad: the encoder refused a bitrate of {}: {refused}", ask.bps); + /// One frame in, one out. The rate and frame rate asked for, if they moved and the + /// encoder can take them. A keyframe is asked for when `ask` says so: with the + /// hardware, ahead of the frame, through the codec properties; with OpenH264, on the + /// encoder it holds. + fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { + match self { + #[cfg(windows)] + Backend::MediaFoundation(encoder) => encoder.encode(planes, ask), + Backend::OpenH264 { encoder, begun, bps, fps } => { + if *begun { + if ask.bps != *bps && ask.bps > 0 { + let mut info = SBitrateInfo { iLayer: SPATIAL_LAYER_ALL, iBitrate: ask.bps as i32 }; + // SAFETY: OpenH264 reads this option as an `SBitrateInfo`, which outlives the call, + // and copies it. The encoder set up on its first frame. + let refused = unsafe { encoder.raw_api().set_option(ENCODER_OPTION_BITRATE, (&raw mut info).cast()) }; + if refused == 0 { + *bps = ask.bps; + } else { + tracing::warn!("mediad: the encoder refused a bitrate of {}: {refused}", ask.bps); + } + } + if ask.fps != *fps && ask.fps > 0. { + let mut rate = ask.fps; + // SAFETY: as above, with a float, which is what this option is read as. + let refused = unsafe { encoder.raw_api().set_option(ENCODER_OPTION_FRAME_RATE, (&raw mut rate).cast()) }; + if refused == 0 { + *fps = ask.fps; + } + } + } + if ask.key && *begun { + encoder.force_intra_frame(); + } + let stamp = Timestamp::from_millis(ask.at.as_millis() as u64); + let encoded = encoder.encode_at(planes, stamp).map_err(|err| err.to_string())?; + let key = matches!(encoded.frame_type(), FrameType::IDR | FrameType::I); + let bytes = encoded.to_vec(); + *begun = true; + Ok(Encoded { bytes, key }) } } - if ask.fps != self.fps && ask.fps > 0. { - let mut fps = ask.fps; - // SAFETY: as above, with a float, which is what this option is read as. - let refused = unsafe { self.encoder.raw_api().set_option(ENCODER_OPTION_FRAME_RATE, (&raw mut fps).cast()) }; - if refused == 0 { - self.fps = ask.fps; + } +} + +impl Coded { + /// `bps` is also the most this encoder ever spends: a tune only goes under it, and so + /// is `fps`. A share's encoder is level 5.1, which holds 1440p at 60. The machine's own + /// hardware takes the job first when Windows lists an encoder for it ([`mf_encoder`]); + /// OpenH264, downloaded at run time, takes every other machine and every refusal, with + /// the words it has always had. + fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + #[cfg(windows)] + if mf_encoder::choose(&mf_encoder::available()) == mf_encoder::Choice::MediaFoundation { + match mf_encoder::Encoder::open(width, height, bps, fps, screen) { + Ok(encoder) => return Ok(Coded { backend: Backend::MediaFoundation(encoder), width, height }), + Err(why) => tracing::warn!("mediad: the Media Foundation encoder could not be set up ({why}); OpenH264 takes over."), } } + let backend = Backend::open_h264(width, height, bps, fps, screen)?; + Ok(Coded { backend, width, height }) } fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { - self.tune(ask); - if ask.key && self.begun { - self.encoder.force_intra_frame(); + self.backend.encode(planes, ask) + } +} + +/// Whether the OpenH264 binary is wanted before this track's first frame: on Windows the +/// machine's own hardware may take the job, and then no download is needed. A decode still +/// wants the binary whatever the send side chose, so `warm` runs on regardless. +fn needs_openh264() -> bool { + #[cfg(windows)] + { + match mf_encoder::available() { + Ok(()) => false, + Err(why) => { + tracing::debug!("mediad: no Media Foundation encoder ({why}); OpenH264 stays the codec."); + true + } } - let stamp = Timestamp::from_millis(ask.at.as_millis() as u64); - let encoded = self.encoder.encode_at(planes, stamp).map_err(|err| err.to_string())?; - let key = matches!(encoded.frame_type(), FrameType::IDR | FrameType::I); - let bytes = encoded.to_vec(); - self.begun = true; - Ok(Encoded { bytes, key }) } + #[cfg(not(windows))] + true } /// The encoder for one track at this size: the one it has, or a new one, which starts with a @@ -198,8 +255,8 @@ impl Cost { impl Outgoing { /// One share frame, or why the screen could not be read. pub fn share(&mut self, ask: Ask) -> Result { - // A capture is wasted until the codec loads. - if self.share.is_none() { + // A capture is wasted until the codec loads, and the hardware needs no load at all. + if self.share.is_none() && needs_openh264() { load_api()?; } let quality = super::share_quality(); @@ -237,7 +294,7 @@ impl Outgoing { /// The camera's two layers, high then low, or why the camera could not be read. A layer with /// no `Ask` is stopped: nothing is spent on it and it comes back empty. pub fn camera(&mut self, high: Option, low: Option) -> Result<(Encoded, Encoded), String> { - if self.high.is_none() || self.low.is_none() { + if (self.high.is_none() || self.low.is_none()) && needs_openh264() { load_api()?; } let began = Instant::now(); @@ -950,12 +1007,13 @@ mod tests { } /// A camera, or a screen, through real OpenH264: `cargo test -p mediad --lib -- --ignored` - /// on a machine that has the binary. + /// on a machine that has the binary. The software backend is opened by name, so a machine + /// with a hardware encoder still tests OpenH264 here. #[test] #[ignore = "needs the OpenH264 binary, which is not in the tree"] fn frames_survive_the_real_codec_in_bitrate_mode_and_the_rate_moves() { let (width, height) = (640usize, 360usize); - let mut outgoing = Coded::open(width as u32, height as u32, CAMERA_HIGH_BPS, FULL_FPS, false).unwrap(); + let mut outgoing = Backend::open_h264(width as u32, height as u32, CAMERA_HIGH_BPS, FULL_FPS, false).unwrap(); let mut incoming = Incoming::default(); let (mut key, mut bytes_high, mut bytes_low) = (false, 0usize, 0usize); for n in 0..90usize { diff --git a/docs/product/06-calls.md b/docs/product/06-calls.md index 755ce335..6c37ed1b 100644 --- a/docs/product/06-calls.md +++ b/docs/product/06-calls.md @@ -12,7 +12,7 @@ A Tiny room joins with sound when the server's audio is up: one send-only Opus m ## Pictures, on a Tiny room -A Tiny call can send a camera and a screen, and it receives one H.264 picture. The codec is OpenH264 constrained baseline, downloaded from Cisco the first time a picture is encoded. If that download fails, the call stays voice. +A Tiny call can send a camera and a screen, and it receives one H.264 picture. On a desktop, what it sends is encoded first by the machine's own hardware: on Windows, Media Foundation lists the hardware H.264 encoders (NVIDIA, Intel, AMD), and the first one that takes the job runs it at the size, frame rate and bitrate the share chose, at constrained baseline, with no download and hardly any CPU. A machine that lists no such encoder, or one that refuses its setup, falls back to OpenH264 constrained baseline, downloaded from Cisco the first time a picture is encoded. If that download fails, the call stays voice. macOS and Linux send through OpenH264 as they always have; whatever arrives is decoded through OpenH264 on every desktop. The room forwards one picture: a share while someone else is sharing, otherwise one layer of another person's camera. The listener does not get their own picture. The client offers three video lines, the first two inactive until asked: a camera as two layers of one picture (640×360 and 320×180), one share fitted inside 1280×720, and one receive line for the picture the room forwards. Both the camera and the share run at 15 frames a second. Nothing plays until someone chooses Watch. From e14e2b97752b6bd54e16ca8da2a16425477d8a31 Mon Sep 17 00:00:00 2001 From: unRekable Date: Tue, 10 Nov 2026 21:21:28 +0100 Subject: [PATCH 2/7] The codec property handle is ICodecAPI, and VARIANT bodies are taken by hand Windows names the interface ICodecAPI, under MediaFoundation in this crate, not IMFCodecAPI. And rustc does not walk a ManuallyDrop on a union field by itself, so each VARIANT body layer is taken explicitly. open_h264 loses the frame size it never used: OpenH264 learns the size from its first frame. --- crates/mediad/src/mf_encoder.rs | 21 +++++++++++++-------- crates/mediad/src/picture/desktop.rs | 9 +++++---- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs index 8004ee38..d5e6d312 100644 --- a/crates/mediad/src/mf_encoder.rs +++ b/crates/mediad/src/mf_encoder.rs @@ -28,7 +28,7 @@ use windows::Win32::Foundation::{RPC_E_CHANGED_MODE, VARIANT_BOOL}; use windows::Win32::Media::MediaFoundation::{ CODECAPI_AVEncCommonLowLatency, CODECAPI_AVEncCommonMeanBitRate, CODECAPI_AVEncCommonRateControlMode, CODECAPI_AVEncMPVDefaultBPictureCount, CODECAPI_AVEncVideoForceKeyFrame, CODECAPI_AVLowLatencyMode, - IMFActivate, IMFCodecAPI, IMFMediaEventGenerator, IMFTransform, MFVideoInterlace_Progressive, + IMFActivate, ICodecAPI, IMFMediaEventGenerator, IMFTransform, MFVideoInterlace_Progressive, MF_E_NO_EVENTS_AVAILABLE, MF_EVENT_FLAG_NO_WAIT, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_DEFAULT_STRIDE, MF_MT_FIXED_SIZE_SAMPLES, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_MPEG2_LEVEL, MF_MT_MPEG2_PROFILE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SAMPLE_SIZE, @@ -129,7 +129,7 @@ pub struct Encoder { events: IMFMediaEventGenerator, /// `None` when the MFT exposes no codec properties, which leaves a keyframe ask and a /// bitrate move without their handle; the picture still encodes. - codec: Option, + codec: Option, width: usize, height: usize, /// The stride promised in the input type: the width rounded up as the hardware wants. @@ -211,7 +211,7 @@ impl Encoder { Err(why) => return Err(why), } - let codec: Option = transform.cast().ok(); + let codec: Option = transform.cast().ok(); if let Some(codec) = &codec { // Both rates are said twice on purpose: once on the output type, which the // MFT reads as it opens, and once through the codec properties, which the @@ -461,11 +461,15 @@ fn rate(fps: f32) -> u64 { /// A `VARIANT` holding a count, which is how the codec properties are read. fn counted(count: u32) -> VARIANT { - // SAFETY: a VARIANT is initialized here and read by the property call it is handed to. + // SAFETY: a VARIANT is initialized here and its body is filled before the property + // call it is handed to reads it. unsafe { let mut value = VariantInit(); - value.Anonymous.Anonymous.vt = VT_UI4; - value.Anonymous.Anonymous.Anonymous.ulVal = count; + // The body is a union behind a ManuallyDrop, which rustc does not walk on its + // own: each layer is taken by hand. + let body = &mut *value.Anonymous.Anonymous; + body.vt = VT_UI4; + body.Anonymous.ulVal = count; value } } @@ -475,8 +479,9 @@ fn yes() -> VARIANT { // SAFETY: as in `counted`, with a boolean. unsafe { let mut value = VariantInit(); - value.Anonymous.Anonymous.vt = VT_BOOL; - value.Anonymous.Anonymous.Anonymous.boolVal = VARIANT_BOOL(-1); + let body = &mut *value.Anonymous.Anonymous; + body.vt = VT_BOOL; + body.Anonymous.boolVal = VARIANT_BOOL(-1); value } } diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index 5d4bed6a..9472d644 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -66,8 +66,9 @@ impl Backend { /// The OpenH264 fallback, opened the way it always was: `bps` is also the most this /// encoder ever spends: [`Backend::encode`] only tunes under it, and so is `fps`. A /// share's encoder is level 5.1, which holds 1440p at 60, and spreads its work over - /// threads: one thread could not encode 1080p inside a frame at 60. - fn open_h264(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + /// threads: one thread could not encode 1080p inside a frame at 60. The size itself is + /// the first frame's, not the opener's. + fn open_h264(bps: u32, fps: f32, screen: bool) -> Result { let api = load_api()?; let threads = if screen { std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 8) as u16 } else { 1 }; let config = EncoderConfig::new() @@ -143,7 +144,7 @@ impl Coded { Err(why) => tracing::warn!("mediad: the Media Foundation encoder could not be set up ({why}); OpenH264 takes over."), } } - let backend = Backend::open_h264(width, height, bps, fps, screen)?; + let backend = Backend::open_h264(bps, fps, screen)?; Ok(Coded { backend, width, height }) } @@ -1013,7 +1014,7 @@ mod tests { #[ignore = "needs the OpenH264 binary, which is not in the tree"] fn frames_survive_the_real_codec_in_bitrate_mode_and_the_rate_moves() { let (width, height) = (640usize, 360usize); - let mut outgoing = Backend::open_h264(width as u32, height as u32, CAMERA_HIGH_BPS, FULL_FPS, false).unwrap(); + let mut outgoing = Backend::open_h264(CAMERA_HIGH_BPS, FULL_FPS, false).unwrap(); let mut incoming = Incoming::default(); let (mut key, mut bytes_high, mut bytes_low) = (false, 0usize, 0usize); for n in 0..90usize { From 33adef007836141d99450a0ef1c02fe9414f592f Mon Sep 17 00:00:00 2001 From: unRekable Date: Tue, 10 Nov 2026 21:52:50 +0100 Subject: [PATCH 3/7] Say why the two NV12 packers differ --- crates/mediad/src/mf_encoder.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs index d5e6d312..6fdf455a 100644 --- a/crates/mediad/src/mf_encoder.rs +++ b/crates/mediad/src/mf_encoder.rs @@ -488,7 +488,9 @@ fn yes() -> VARIANT { /// `planes` as NV12 into `out`, Y rows at `stride` and the two chroma planes interleaved /// under them. The padding rows and bytes stay what the buffer was made with, and a buffer -/// too short takes what fits and nothing past it. +/// too short takes what fits and nothing past it. The phone's `convert::nv12` packs the +/// same planes tightly into a `VideoFrame`; this one is the shape the input type promises +/// the machine's encoder: its stride, in the buffer its sample was given. fn nv12(planes: &Planes, stride: usize, out: &mut [u8]) { let (width, height) = (planes.width, planes.height); for row in 0..height { From 6818795f2933f6bcaed0a52040fb067547dff323 Mon Sep 17 00:00:00 2001 From: unRekable Date: Tue, 10 Nov 2026 23:29:14 +0100 Subject: [PATCH 4/7] Relay: the gate scenario drives the Media Foundation encoder's selection and helper tests --- .../hardware-encoder-fallback/README.md | 13 ++++++++ .../hardware-encoder-fallback/run.sh | 30 +++++++++++++++++++ .../hardware-encoder-fallback/scenario.json | 10 +++++++ 3 files changed, 53 insertions(+) create mode 100644 e2e/scenarios/hardware-encoder-fallback/README.md create mode 100644 e2e/scenarios/hardware-encoder-fallback/run.sh create mode 100644 e2e/scenarios/hardware-encoder-fallback/scenario.json diff --git a/e2e/scenarios/hardware-encoder-fallback/README.md b/e2e/scenarios/hardware-encoder-fallback/README.md new file mode 100644 index 00000000..3d3eaa7c --- /dev/null +++ b/e2e/scenarios/hardware-encoder-fallback/README.md @@ -0,0 +1,13 @@ +# Hardware encoder fallback + +Run `sh e2e/scenarios/hardware-encoder-fallback/run.sh`. It needs no devices and no +network: the scenario drives the selection logic and the pure NV12/Annex B helpers of +`crates/mediad/src/mf_encoder.rs`. + +On the base the scenario fails: the module does not exist, so cargo reports no matching +tests. With the branch the Media Foundation hardware encoder is tried first (when the +machine lists one that takes NV12 and answers H.264) and the OpenH264 runtime download +stays the fallback with its own covered path. + +The on-hardware record (the lab driving a real share over a call) needs a machine with a +hardware MFT; that recording is the released-client proof this scenario points to. diff --git a/e2e/scenarios/hardware-encoder-fallback/run.sh b/e2e/scenarios/hardware-encoder-fallback/run.sh new file mode 100644 index 00000000..27cdf200 --- /dev/null +++ b/e2e/scenarios/hardware-encoder-fallback/run.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# The Media Foundation hardware encoder's selection and helpers, unit-tested on Windows (the +# module is cfg(windows), so this driver runs where it compiles). No network, no devices. +set -eu +cd "$(dirname "$0")/../../.." +log=$(mktemp) +trap 'rm -f "$log"' EXIT HUP INT TERM +run() { + name=$1 + printf '%s\n' "media foundation encoder: $name" + if ! cargo test -p mediad --lib "mf_encoder::tests::$name" -- --exact --nocapture --test-threads=1 >"$log" 2>&1; then + cat "$log" + return 1 + fi + cat "$log" + grep -Fq "running 1 test" "$log" + grep -Fq "test mf_encoder::tests::$name ... ok" "$log" + if grep -Eq '(^|[[:space:]])skipped:' "$log"; then + echo 'FAIL: a fixture skipped' >&2 + return 1 + fi +} +run a_clean_probe_means_hardware_and_any_failure_means_the_download +run nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved +run nv12_of_a_short_buffer_writes_nothing_past_it +run annex_b_already_annex_b_is_untouched +run length_prefixed_units_come_out_behind_start_codes +run a_truncated_or_empty_stream_never_runs_past_itself +run sizes_and_rates_are_packed_width_over_height_and_thousandths +printf '%s\n' 'PASS: the hardware encoder is chosen when present and OpenH264 stays the fallback.' diff --git a/e2e/scenarios/hardware-encoder-fallback/scenario.json b/e2e/scenarios/hardware-encoder-fallback/scenario.json new file mode 100644 index 00000000..e4b07a2e --- /dev/null +++ b/e2e/scenarios/hardware-encoder-fallback/scenario.json @@ -0,0 +1,10 @@ +{ + "id": "hardware-encoder-fallback", + "title": "A camera or screen share encodes through the machine's hardware encoder when present and falls back to the OpenH264 download otherwise", + "platforms": ["windows"], + "needs": [], + "release": false, + "driver": "run.sh", + "proof_scope": "encoder selection and helpers, unit-level; the on-hardware record needs a machine with a hardware MFT", + "native_client_proof": "the released client shows screen share and camera running on the machine's own hardware encoder, with the OpenH264 download as the fallback" +} From 53484dc2b93a38a3633441ec3e7141addff836cb Mon Sep 17 00:00:00 2001 From: unRekable Date: Wed, 11 Nov 2026 00:21:39 +0100 Subject: [PATCH 5/7] The output type first, the wire held to the baseline, and a mid-share failure falls back to OpenH264 --- crates/mediad/src/mf_encoder.rs | 1077 +++++++++++++---- crates/mediad/src/picture/desktop.rs | 103 +- .../hardware-encoder-fallback/README.md | 23 +- .../hardware-encoder-fallback/run.sh | 34 +- .../hardware-encoder-fallback/scenario.json | 2 +- 5 files changed, 960 insertions(+), 279 deletions(-) diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs index 6fdf455a..12781606 100644 --- a/crates/mediad/src/mf_encoder.rs +++ b/crates/mediad/src/mf_encoder.rs @@ -4,21 +4,33 @@ //! A hardware encoder is an asynchronous Media Foundation transform (MFT): it is found with //! `MFTEnumEx` over the registered hardware encoders, woken by `MF_TRANSFORM_ASYNC_UNLOCK`, //! fed NV12 frames through `ProcessInput`, and drained through its event queue -//! (`METransformNeedInput`, `METransformHaveOutput`) with `ProcessOutput`. The rate is CBR -//! at the bitrate the track asked for (`MF_MT_AVG_BITRATE` and +//! (`METransformNeedInput`, `METransformHaveOutput`) with `ProcessOutput`. The output type +//! is negotiated first, at the constrained baseline the SFU takes, and what the transform +//! kept is read back and held to that: a driver that would put Main or High on the wire, +//! or one that names no profile at all, is refused in its own words and OpenH264 takes the +//! job. The rate is CBR at the bitrate the track asked for (`MF_MT_AVG_BITRATE` and //! `CODECAPI_AVEncCommonRateControlMode`), a keyframe is asked for with -//! `CODECAPI_AVEncVideoForceKeyFrame`, and the output is normalized to the Annex B byte -//! stream the SFU and the OpenH264 decoder take. A frame whose bits are not out yet comes -//! back as an empty [`Encoded`], which the share already treats as a skip; they leave on a -//! later frame's pump. +//! `CODECAPI_AVEncVideoForceKeyFrame`, a frame is a keyframe only when its sample says +//! `MFSampleExtension_CleanPoint`, and the output is normalized to the Annex B byte stream +//! the SFU and the OpenH264 decoder take. A frame whose bits are not out yet comes back as +//! an empty [`Encoded`], which the share already treats as a skip; they leave on a later +//! frame's pump. The first frame may wait the encoder's warmup for its bits; every later +//! one waits a far shorter moment, so a driver that holds frames back never stalls the +//! capture thread. +//! +//! The asks of the transform are narrowed to the [`Mft`] trait, and the transform comes +//! from a [`Source`]: [`Hardware`] is the machine's own, and the tests hold fakes against +//! the same words, which is how the open, the profile refusal, the keyframe flag and the +//! frame loop are proven without hardware. //! //! Everything here runs where the encoder was opened: the share's or camera's capture //! thread, COM in the Multi Threaded Apartment, Media Foundation started without its -//! sockets. `Outgoing` only ever moves between threads before an encoder exists, and MTA -//! objects are callable from every thread of the process, so the `Send` on [`Encoder`] is -//! honest. What the encoder keeps between frames is its bitrate, its frame rate, and -//! whether it has an input slot to fill; a change of frame size is not a tune but a new -//! encoder (`coded` in `desktop`), as with OpenH264. +//! sockets. Both are taken up by [`platform`] and let go by the [`Platform`] guard the +//! encoder holds, after the transform is. `Outgoing` only ever moves between threads before +//! an encoder exists, and MTA objects are callable from every thread of the process, so the +//! `Send` on [`Encoder`] is honest. What the encoder keeps between frames is its bitrate, +//! its frame rate, and whether it has an input slot to fill; a change of frame size is not +//! a tune but a new encoder (`coded` in `desktop`), as with OpenH264. use std::ptr::null_mut; use std::sync::OnceLock; @@ -28,31 +40,41 @@ use windows::Win32::Foundation::{RPC_E_CHANGED_MODE, VARIANT_BOOL}; use windows::Win32::Media::MediaFoundation::{ CODECAPI_AVEncCommonLowLatency, CODECAPI_AVEncCommonMeanBitRate, CODECAPI_AVEncCommonRateControlMode, CODECAPI_AVEncMPVDefaultBPictureCount, CODECAPI_AVEncVideoForceKeyFrame, CODECAPI_AVLowLatencyMode, - IMFActivate, ICodecAPI, IMFMediaEventGenerator, IMFTransform, MFVideoInterlace_Progressive, + IMFActivate, ICodecAPI, IMFMediaEventGenerator, IMFMediaType, IMFTransform, MFVideoInterlace_Progressive, MF_E_NO_EVENTS_AVAILABLE, MF_EVENT_FLAG_NO_WAIT, MF_MT_ALL_SAMPLES_INDEPENDENT, MF_MT_AVG_BITRATE, MF_MT_DEFAULT_STRIDE, MF_MT_FIXED_SIZE_SAMPLES, MF_MT_FRAME_RATE, MF_MT_FRAME_SIZE, MF_MT_INTERLACE_MODE, MF_MT_MAJOR_TYPE, MF_MT_MPEG2_LEVEL, MF_MT_MPEG2_PROFILE, MF_MT_PIXEL_ASPECT_RATIO, MF_MT_SAMPLE_SIZE, - MF_MT_SUBTYPE, MF_TRANSFORM_ASYNC_UNLOCK, MF_VERSION, METransformHaveOutput, METransformNeedInput, - MFTEnumEx, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, MFT_ENUM_FLAG_SORTANDFILTER, - MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, MFT_MESSAGE_NOTIFY_END_STREAMING, - MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER, MFT_REGISTER_TYPE_INFO, MFCreateMediaType, - MFCreateMemoryBuffer, MFCreateSample, MFMediaType_Video, MFStartup, MFSTARTUP_NOSOCKET, - MFVideoFormat_H264, MFVideoFormat_NV12, MFSampleExtension_CleanPoint, eAVEncCommonRateControlMode_CBR, - eAVEncH264VLevel3_1, eAVEncH264VLevel5_1, eAVEncH264VProfile_ConstrainedBase, + MF_MT_SUBTYPE, MF_TRANSFORM_ASYNC_UNLOCK, MF_VERSION, MFShutdown, METransformHaveOutput, + METransformNeedInput, MFTEnumEx, MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE, + MFT_ENUM_FLAG_SORTANDFILTER, MFT_MESSAGE_COMMAND_FLUSH, MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, + MFT_MESSAGE_NOTIFY_END_STREAMING, MFT_MESSAGE_NOTIFY_START_OF_STREAM, MFT_OUTPUT_DATA_BUFFER, + MFT_REGISTER_TYPE_INFO, MFCreateMediaType, MFCreateMemoryBuffer, MFCreateSample, MFMediaType_Video, + MFStartup, MFSTARTUP_NOSOCKET, MFVideoFormat_H264, MFVideoFormat_NV12, MFSampleExtension_CleanPoint, + eAVEncCommonRateControlMode_CBR, eAVEncH264VLevel3_1, eAVEncH264VLevel5_1, eAVEncH264VProfile_Base, + eAVEncH264VProfile_ConstrainedBase, }; -use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoTaskMemFree}; +use windows::Win32::System::Com::{COINIT_MULTITHREADED, CoInitializeEx, CoTaskMemFree, CoUninitialize}; use windows::Win32::System::Variant::{VT_BOOL, VT_UI4, VARIANT, VariantInit}; use crate::picture::{Ask, Encoded, Planes}; -/// How long one frame waits for the machine's encoder to answer. Hardware encoders in low -/// latency answer inside a frame's time; when one does not, the frame goes out empty and -/// its bits leave on the next pump, the way a skip already does. +/// How long the first frame waits for the machine's encoder to answer, which is the one +/// frame that may cover a driver's warmup. Hardware encoders in low latency answer inside +/// a frame's time; when one does not, the frame goes out empty and its bits leave on the +/// next pump, the way a skip already does. const ANSWER_WAIT: Duration = Duration::from_millis(120); +/// How long every frame after the first waits for its own bits: far less than a frame's +/// time, so a slow driver or a lookahead holds the capture thread no longer than this. +const OUTPUT_WAIT: Duration = Duration::from_millis(10); /// How often the event queue is looked at while waiting for that answer. const POLL: Duration = Duration::from_millis(1); /// The NV12 stride promised to the encoder, the width rounded up as the hardware wants it. const STRIDE_ALIGN: usize = 16; +/// The profiles the SFU is told to expect, as Media Foundation names them: +/// `eAVEncH264VProfile_Base` (66, plain baseline) and `eAVEncH264VProfile_ConstrainedBase` +/// (256). Main (77) and High (100) are refused, and so is a type that names none. +const BASELINE: u32 = eAVEncH264VProfile_Base.0 as u32; +const CONSTRAINED_BASELINE: u32 = eAVEncH264VProfile_ConstrainedBase.0 as u32; /// Which backend a track's encoder takes: the machine's hardware on a clean probe, the /// OpenH264 download on any failure. The one place that decision lives, so tests can hold @@ -71,17 +93,36 @@ pub fn choose(probe: &Result<(), String>) -> Choice { /// Whether this machine has a hardware H.264 encoder Media Foundation can drive, asked once /// and remembered for the process: the share and camera paths read this to keep the /// OpenH264 download off a machine that will never need it, and [`Encoder::open`] asks the -/// system again before it opens. +/// system again before it opens. The probe runs on a thread of its own, so the answer is +/// the machine's and not the asking thread's apartment; an answer that was never really +/// taken (the probe's thread would not start) is not remembered and is asked again at the +/// next open. pub fn available() -> Result<(), String> { static AVAILABLE: OnceLock> = OnceLock::new(); - AVAILABLE.get_or_init(probe).clone() + match probe_thread() { + Ok(answer) => AVAILABLE.get_or_init(|| answer).clone(), + Err(why) => Err(why), + } +} + +/// The probe on a thread of its own. A fresh thread holds no COM apartment, so the probe's +/// own `CoInitializeEx` decides how it runs and an `RPC_E_CHANGED_MODE` from elsewhere +/// cannot answer for the machine. The outer `Err` is the thread's, not the machine's, and +/// is never cached. +fn probe_thread() -> Result, String> { + let asked = std::thread::Builder::new().name("mediad-mf-probe".into()).spawn(probe) + .map_err(|err| format!("the encoder probe's thread would not start: {err}"))?; + match asked.join() { + Ok(answer) => Ok(answer), + Err(_) => Err("the encoder probe stopped unexpectedly.".to_owned()), + } } /// One look at what the machine lists, without opening anything. Fails in its own words: a /// machine with no hardware MFT shares through OpenH264, and so does one whose COM or Media /// Foundation would not start. fn probe() -> Result<(), String> { - platform()?; + let _platform = platform()?; if enumerate() == 0 { return Err("This machine lists no hardware H.264 encoder.".to_owned()); } @@ -89,24 +130,72 @@ fn probe() -> Result<(), String> { } /// How many hardware H.264 encoders this machine lists, taking NV12 and answering H.264. +/// Every activate the listing handed out is let go here. fn enumerate() -> u32 { + let (list, count) = match enum_mfts() { + Ok(found) => found, + Err(_) => (null_mut(), 0), + }; + // SAFETY: no entry is kept: every activate in the array is released, then the array. + unsafe { let _ = release(list, count, None); } + count +} + +/// The hardware H.264 encoders MFTEnumEx lists, as the raw array it handed back and its +/// count. A failed listing is an empty one: the answer is the same machine with none. +fn enum_mfts() -> Result<(*mut Option, u32), windows::core::Error> { let takes = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_NV12 }; let answers = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_H264 }; let mut list: *mut Option = null_mut(); let mut count = 0u32; - // SAFETY: both registration types outlive the call, and the list it hands back is freed - // whole below, after the count is read. Nothing here keeps a handle into it. - let _ = unsafe { - MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, Some(&takes), Some(&answers), &mut list, &mut count) - }; - // SAFETY: the list is MFTEnumEx's own allocation, freed whole; no element is read. + // SAFETY: both registration types outlive the call, and the list it hands back is + // freed whole by `release`, after the entries are let go one by one. Nothing here + // keeps a handle into it. + unsafe { MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, Some(&takes), Some(&answers), &mut list, &mut count) }?; + Ok((list, count)) +} + +/// Every activate in MFTEnumEx's array let go, then the array itself. Which entry `keep` +/// moves out first is the caller's one take, handed back; the rest are released whatever +/// happens with it. +/// +/// SAFETY: `list` is MFTEnumEx's own allocation holding `count` entries (or null with no +/// count), and no other reference into it exists. +unsafe fn release(list: *mut Option, count: u32, keep: Option) -> Option { + let mut kept = None; + for at in 0..count as usize { + let entry = list.add(at); + if keep == Some(at) { + // SAFETY: the entry is read once, moved out of the array before it goes. + kept = unsafe { std::ptr::read(entry) }; + } else { + // SAFETY: the entry is MFTEnumEx's own, dropped here to release the activate. + unsafe { std::ptr::drop_in_place(entry) }; + } + } + // SAFETY: the array is MFTEnumEx's own allocation, freed whole. unsafe { CoTaskMemFree(Some(list.cast::())) }; - count + kept +} + +/// COM in the Multi Threaded Apartment and Media Foundation, taken up for one encoder or +/// one probe and let go when the guard goes, each undone only if this guard took it up. +/// Media Foundation is per process and keeps count, so an encoder closing beside another +/// still running leaves that one alone; the apartment is per thread and keeps count the +/// same way. The guard must drop on the thread `platform` ran on, which is the encoder's +/// whole life (the share's or the camera's capture thread), and drops after the encoder's +/// transform, which is the field before it, so nothing is asked of Media Foundation after +/// it let go. +struct Platform { + /// Whether this guard's thread was brought into the MTA by [`platform`]. + com: bool, + /// Whether this guard started Media Foundation. + mf: bool, } /// COM in the Multi Threaded Apartment, per thread, and Media Foundation, per process. /// Both keep count, so an encoder thread that meets another's work answers at once. -fn platform() -> Result<(), String> { +fn platform() -> Result { // SAFETY: the apartment call is the documented per-thread init; its result is checked. let started = unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) }; if started.is_err() { @@ -116,246 +205,235 @@ fn platform() -> Result<(), String> { format!("COM would not start for the encoder: {started}") }); } - // The init is never undone: the capture thread outlives the encoder by a frame at most, - // and Media Foundation's own teardown races nothing that outlives it here. + let mut guard = Platform { com: true, mf: false }; + // The init is undone by the guard when this open ends, on the thread that took it. // SAFETY: the version and the no-sockets flag are the constants Media Foundation reads. - unsafe { MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET) }.map_err(|err| format!("Media Foundation would not start: {err}")) + if let Err(err) = unsafe { MFStartup(MF_VERSION, MFSTARTUP_NOSOCKET) } { + return Err(format!("Media Foundation would not start: {err}")); + } + guard.mf = true; + Ok(guard) } -/// The machine's hardware H.264 encoder, held as the async MFT, its event queue, and the -/// codec properties that survive a frame. -pub struct Encoder { - transform: IMFTransform, - events: IMFMediaEventGenerator, - /// `None` when the MFT exposes no codec properties, which leaves a keyframe ask and a - /// bitrate move without their handle; the picture still encodes. - codec: Option, - width: usize, - height: usize, - /// The stride promised in the input type: the width rounded up as the hardware wants. - stride: usize, - /// One NV12 frame at that stride: the Y plane and the two interleaved chroma planes. - frame: usize, - /// What the encoder holds now, so a tune is only sent when the ask moved. - bps: u32, - fps: f32, - /// Whether the encoder has an input slot open: a `METransformNeedInput` seen and not - /// yet fed. - wants_input: bool, +impl Drop for Platform { + fn drop(&mut self) { + if self.mf { + // SAFETY: Media Foundation was started by this guard's platform call. + unsafe { let _ = MFShutdown(); } + } + if self.com { + // SAFETY: the apartment was taken by this guard's platform call, on the thread + // the guard is dropped on. + unsafe { CoUninitialize() } + } + } } -/// SAFETY: the encoder is opened, driven, and dropped on one thread (the share's or the -/// camera's capture thread); `Outgoing` moves only before any encoder exists. The COM -/// apartment is the MTA and Media Foundation objects answer from every thread of the -/// process, so even a move after the fact would stay within what the platform allows. -unsafe impl Send for Encoder {} +#[cfg(test)] +impl Platform { + /// A guard for nothing started, so the tests assemble an encoder without touching the + /// machine's COM, and the drop has nothing to undo. + fn unstarted() -> Platform { + Platform { com: false, mf: false } + } +} -impl Encoder { - /// The first hardware encoder that takes NV12 and answers H.264, set up for CBR at - /// `bps` and at most `fps`, level 5.1 for a screen and 3.1 for a camera, at the profile - /// the SFU takes. Its own words when there is no such encoder or one would not take its - /// setup; the caller falls back to OpenH264 on any of them. - pub fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { - platform()?; - let (width, height) = (width.max(2) as usize, height.max(2) as usize); - let stride = (width + STRIDE_ALIGN - 1) & !(STRIDE_ALIGN - 1); - let frame = stride * height * 3 / 2; +/// What the encoder's event queue said: an input slot opened, or a frame is out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Event { + NeedInput, + HaveOutput, +} - let mut list: *mut Option = null_mut(); - let mut count = 0u32; - let takes = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_NV12 }; - let answers = MFT_REGISTER_TYPE_INFO { guidMajorType: MFMediaType_Video, guidSubtype: MFVideoFormat_H264 }; - // SAFETY: as in `enumerate`, the registration types outlive the call and the list is - // freed below, after its first entry has been taken out of it. - unsafe { MFTEnumEx(MFT_CATEGORY_VIDEO_ENCODER, MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, Some(&takes), Some(&answers), &mut list, &mut count) } - .map_err(|err| format!("the hardware encoders could not be listed: {err}"))?; - let first = if count > 0 { - // SAFETY: `count` says the list holds at least one entry, and reading it moves - // the interface out of MFTEnumEx's allocation before that allocation is freed. - Some(unsafe { std::ptr::read(list.add(0)) }) - } else { - None - }; - // SAFETY: as above, the list is freed whole. - unsafe { CoTaskMemFree(Some(list.cast::())) }; - let Some(activate) = first.flatten() else { +/// One frame out of the transform: its bytes as the Annex B stream, and what the sample +/// said about being a clean point, if it said anything. +#[derive(Debug, Clone, PartialEq)] +struct Taken { + bytes: Vec, + clean: Option, +} + +/// A frame is a keyframe only when the sample says it is a clean point, and says it +/// loudly: an attribute that is merely there, or there as zero, is a delta frame. +fn clean_point_is_key(clean: Option) -> bool { + clean.unwrap_or(0) != 0 +} + +/// The one hardware transform an [`Encoder`] drives, narrowed to the words open and encode +/// ask of it. The real one is the machine's MFT in COM, handed over by [`Hardware`]; the +/// tests hold fakes against the same words, which is how the open, the fallback and the +/// keyframe flag are held without hardware. +trait Mft: Send { + /// Wake the async transform for event-driven use, before any type is set. + fn unlock(&mut self) -> Result<(), String>; + /// The output ask: H.264 at the bitrate, size and rate of the track, constrained + /// baseline at the level the track asked for. Answers the profile of the type the + /// transform kept, so what goes on the wire can be held to what was promised; `None` + /// is a kept type that names no profile. + fn set_output_type(&mut self, width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result, String>; + /// The input ask: NV12 at the promised stride, whole frames, at the size and rate. + fn set_input_type(&mut self, width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result<(), String>; + /// The codec properties settled for the stream: CBR at `bps`, low latency, no B + /// pictures. A refusal is said in the log and lived with, never a failure. + fn configure(&mut self, bps: u32); + /// Streaming begun. + fn begin(&mut self) -> Result<(), String>; + /// The next event that asks something of the encoder, skipping the kinds a share needs + /// nothing of; `None` when the queue has nothing more now. + fn event(&mut self) -> Result, String>; + /// One NV12 frame in, at `at_100ns` since the stream began and lasting + /// `lasting_100ns`. + fn give(&mut self, planes: &Planes, stride: usize, frame: usize, at_100ns: i64, lasting_100ns: i64) -> Result<(), String>; + /// One frame out, or `None` when the transform has nothing out after all. + fn take(&mut self) -> Result, String>; + /// The bitrate moved; answers whether the transform took it. + fn tune(&mut self, bps: u32) -> bool; + /// A keyframe for the next frame in. + fn force_key(&mut self); + /// Flush and end streaming, at close. + fn end(&mut self); +} + +/// Where an encoder's transform comes from: [`Hardware`], the machine's first hardware MFT +/// that takes NV12 and answers H.264, and the tests' fakes. +trait Source { + fn activate(&self) -> Result, String>; +} + +/// The machine's own encoders, as MFTEnumEx lists them: the first hardware MFT that takes +/// NV12 and answers H.264, activated, shut down in the activate's own hands, with every +/// activate the listing handed out let go. +struct Hardware; + +impl Source for Hardware { + fn activate(&self) -> Result, String> { + let (list, count) = enum_mfts().map_err(|err| format!("the hardware encoders could not be listed: {err}"))?; + // SAFETY: entry 0 is taken out for this encoder; every other entry and the array + // itself are released here, whatever happens below. + let chosen = unsafe { release(list, count, Some(0)) }; + let Some(activate) = chosen else { return Err("This machine lists no hardware H.264 encoder.".to_owned()); }; // SAFETY: the activation object hands out the transform it stands for, once. let transform: IMFTransform = unsafe { activate.ActivateObject() } .map_err(|err| format!("the hardware encoder could not be started: {err}"))?; + // The activate need not keep the encoder's setup alive once the transform is out: + // its shutdown hands the lifetime to the transform this encoder holds, the way + // MFTEnumEx's own contract asks. + // SAFETY: the activate is still owned here. + unsafe { let _ = activate.ShutdownObject(); } + drop(activate); + // SAFETY: both casts are on the transform this open created. + let events: IMFMediaEventGenerator = unsafe { transform.cast() } + .map_err(|err| format!("the hardware encoder has no event queue: {err}"))?; + // `codec` is `None` when the MFT exposes no codec properties, which leaves a + // keyframe ask and a bitrate move without their handle; the picture still encodes. + let codec: Option = unsafe { transform.cast() }.ok(); + Ok(Box::new(RealMft { transform, events, codec })) + } +} - // SAFETY: from here on every call is on interfaces this open creates or this - // encoder holds, whose pointers the calls only read. - unsafe { - // A hardware encoder answers on its own clock. The unlock tells Media - // Foundation this client will drive it through its event queue, before any - // type is set. - let attributes = transform.GetAttributes().map_err(|err| format!("the hardware encoder told nothing about itself: {err}"))?; - attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1).map_err(|err| format!("the hardware encoder could not be unlocked: {err}"))?; - - let input = input_type(width, height, stride, frame, fps)?; - transform.SetInputType(0, &input, 0).map_err(|err| format!("the hardware encoder would not take NV12: {err}"))?; - - match output_type(width, height, bps, fps, screen, true) { - Ok(output) => { - if transform.SetOutputType(0, &output, 0).is_err() { - // A profile or level this encoder does not keep is its own to pick: - // ask again without them, and say so once. - let plain = output_type(width, height, bps, fps, screen, false)?; - transform.SetOutputType(0, &plain, 0) - .map_err(|err| format!("the hardware encoder would not take an H.264 output: {err}"))?; - tracing::debug!("mediad: the hardware encoder chose its own H.264 profile."); - } - } - Err(why) => return Err(why), - } +/// The machine's encoder as COM hands it over: the async MFT, its event queue, and the +/// codec properties that survive a frame. +struct RealMft { + transform: IMFTransform, + events: IMFMediaEventGenerator, + codec: Option, +} - let codec: Option = transform.cast().ok(); - if let Some(codec) = &codec { - // Both rates are said twice on purpose: once on the output type, which the - // MFT reads as it opens, and once through the codec properties, which the - // rate control holds for the frames after. A refusal is logged and lived - // with: the encoder still answers, at a rate of its own choosing. - if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonRateControlMode, &counted(eAVEncCommonRateControlMode_CBR.0 as u32)) { - tracing::debug!("mediad: the hardware encoder took no CBR rate control: {err}"); - } - if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(bps)) { - tracing::debug!("mediad: the hardware encoder took no mean bitrate: {err}"); - } - if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonLowLatency, &yes()) { - tracing::debug!("mediad: the hardware encoder took no low latency switch: {err}"); - } - if let Err(err) = codec.SetValue(&CODECAPI_AVLowLatencyMode, &yes()) { - tracing::debug!("mediad: the hardware encoder took no low latency mode: {err}"); - } - if let Err(err) = codec.SetValue(&CODECAPI_AVEncMPVDefaultBPictureCount, &counted(0)) { - tracing::debug!("mediad: the hardware encoder took no B picture count: {err}"); - } - } +/// SAFETY: the MFT is opened, driven, and dropped on one thread (the share's or the +/// camera's capture thread); `Outgoing` moves only before any encoder exists. The COM +/// apartment is the MTA and Media Foundation objects answer from every thread of the +/// process, so even a move after the fact would stay within what the platform allows. +unsafe impl Send for RealMft {} - let events: IMFMediaEventGenerator = transform.cast().map_err(|err| format!("the hardware encoder has no event queue: {err}"))?; - transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0).map_err(|err| format!("the hardware encoder would not start streaming: {err}"))?; - transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0).map_err(|err| format!("the hardware encoder would not start streaming: {err}"))?; - - let mut encoder = Encoder { transform, events, codec, width, height, stride, frame, bps, fps, wants_input: false }; - // The first input slot is usually open before the first frame; wait a moment - // for it, so the first frame of a share is fed at once and not dropped. - let until = Instant::now() + ANSWER_WAIT; - while !encoder.wants_input && Instant::now() < until { - encoder.pump(&mut Vec::new(), &mut false)?; - if !encoder.wants_input { - std::thread::sleep(POLL); - } - } - Ok(encoder) +impl Mft for RealMft { + fn unlock(&mut self) -> Result<(), String> { + // SAFETY: the attributes belong to the transform this encoder holds. + unsafe { + let attributes = self.transform.GetAttributes() + .map_err(|err| format!("the hardware encoder told nothing about itself: {err}"))?; + attributes.SetUINT32(&MF_TRANSFORM_ASYNC_UNLOCK, 1) + .map_err(|err| format!("the hardware encoder could not be unlocked: {err}")) } } - /// One frame in, whatever is out. A frame whose bits are still on their way comes back - /// empty, like a skip; an encoded frame comes back marked when it is a keyframe, so - /// the share sends it as one. - pub fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { - if planes.width != self.width || planes.height != self.height { - return Err(format!("The encoder was opened for {}×{} and handed {}×{}.", self.width, self.height, planes.width, planes.height)); - } - self.tune(ask); - if ask.key { - self.request_key(); - } - // What is already out, before this frame goes in: the machine may be a frame ahead. - let mut bytes = Vec::new(); - let mut key = false; - self.pump(&mut bytes, &mut key)?; - if self.wants_input { - self.submit(planes, ask)?; - // A bounded moment for this frame's own bits. When they are not out in time, - // the empty frame goes and the pump catches them next time. - let until = Instant::now() + ANSWER_WAIT; - while bytes.is_empty() && Instant::now() < until { - self.pump(&mut bytes, &mut key)?; - if bytes.is_empty() { - std::thread::sleep(POLL); - } - } - } - if bytes.is_empty() { - key = false; + fn set_output_type(&mut self, width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result, String> { + // SAFETY: the media type is built here and set on the transform this encoder + // holds; what the transform kept is read back before the call returns. + unsafe { + let output = output_type(width, height, bps, fps, screen)?; + self.transform.SetOutputType(0, &output, 0) + .map_err(|err| format!("the hardware encoder would not take an H.264 output: {err}"))?; + let kept = self.transform.GetOutputCurrentType(0) + .map_err(|err| format!("the hardware encoder would not say what it kept of the H.264 output: {err}"))?; + Ok(kept.GetUINT32(&MF_MT_MPEG2_PROFILE).ok()) } - Ok(Encoded { bytes, key }) } - /// The rate asked for, if it moved. The frame rate is set at open and is not moved - /// again; a moving ask for it is a no-op here. - fn tune(&mut self, ask: &Ask) { - if ask.bps == self.bps || ask.bps == 0 { - return; - } - let Some(codec) = &self.codec else { - return; - }; - // SAFETY: the property is read through the codec handle this encoder holds, and - // the variant outlives the call. - match unsafe { codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(ask.bps)) } { - Ok(()) => self.bps = ask.bps, - Err(err) => tracing::warn!("mediad: the hardware encoder refused a bitrate of {}: {err}", ask.bps), + fn set_input_type(&mut self, width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result<(), String> { + // SAFETY: the media type is created here and set before it drops. + unsafe { + let input = input_type(width, height, stride, frame, fps)?; + self.transform.SetInputType(0, &input, 0) + .map_err(|err| format!("the hardware encoder would not take NV12: {err}")) } } - /// A keyframe for the next frame in. The property carries no value; setting it is the - /// ask. An encoder without codec properties cannot be asked, and keeps its own - /// keyframe pace. - fn request_key(&mut self) { + fn configure(&mut self, bps: u32) { let Some(codec) = &self.codec else { return; }; - // SAFETY: as in `tune`, with a property whose value is ignored. - if let Err(err) = unsafe { codec.SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &counted(0)) } { - tracing::debug!("mediad: the hardware encoder refused a keyframe: {err}"); + // Both rates are said twice on purpose: once on the output type, which the + // MFT reads as it opens, and once through the codec properties, which the + // rate control holds for the frames after. A refusal is logged and lived + // with: the encoder still answers, at a rate of its own choosing. + // SAFETY: each property is read through the codec handle this encoder holds, and + // each variant outlives the call it is handed to. + unsafe { + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonRateControlMode, &counted(eAVEncCommonRateControlMode_CBR.0 as u32)) { + tracing::debug!("mediad: the hardware encoder took no CBR rate control: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(bps)) { + tracing::debug!("mediad: the hardware encoder took no mean bitrate: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncCommonLowLatency, &yes()) { + tracing::debug!("mediad: the hardware encoder took no low latency switch: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVLowLatencyMode, &yes()) { + tracing::debug!("mediad: the hardware encoder took no low latency mode: {err}"); + } + if let Err(err) = codec.SetValue(&CODECAPI_AVEncMPVDefaultBPictureCount, &counted(0)) { + tracing::debug!("mediad: the hardware encoder took no B picture count: {err}"); + } } } - /// This frame as a sample, its planes packed NV12 at the promised stride. - fn submit(&mut self, planes: &Planes, ask: &Ask) -> Result<(), String> { - // SAFETY: the sample is created here, filled here, and handed to the transform, - // which owns its reference from `ProcessInput` on; the buffer pointer is used - // within its lock below and gone before the lock is let go. + fn begin(&mut self) -> Result<(), String> { + // SAFETY: the transform is owned here; both messages are its own. unsafe { - let buffer = MFCreateMemoryBuffer(self.frame as u32).map_err(|err| format!("the frame could not be given room: {err}"))?; - let mut data: *mut u8 = null_mut(); - buffer.Lock(&mut data, None, None).map_err(|err| format!("the frame could not be written: {err}"))?; - nv12(planes, self.stride, std::slice::from_raw_parts_mut(data, self.frame)); - let _ = buffer.SetCurrentLength(self.frame as u32); - buffer.Unlock().map_err(|err| format!("the frame could not be handed over: {err}"))?; - - let sample = MFCreateSample().map_err(|err| format!("the frame could not be wrapped: {err}"))?; - sample.AddBuffer(&buffer).map_err(|err| format!("the frame could not be wrapped: {err}"))?; - // The ask's clock is milliseconds; Media Foundation's is 100 nanoseconds. - sample.SetSampleTime((ask.at.as_millis() as i64).saturating_mul(10_000)).map_err(|err| format!("the frame's time was refused: {err}"))?; - let lasting = (10_000_000.0 / self.fps.max(1.0)) as i64; - sample.SetSampleDuration(lasting.max(1)).map_err(|err| format!("the frame's length was refused: {err}"))?; - self.transform.ProcessInput(0, &sample, 0).map_err(|err| format!("the hardware encoder would not take the frame: {err}"))?; - self.wants_input = false; - Ok(()) + self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0) + .map_err(|err| format!("the hardware encoder would not start streaming: {err}"))?; + self.transform.ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0) + .map_err(|err| format!("the hardware encoder would not start streaming: {err}")) } } - /// Everything the event queue holds now, without waiting: frames out, and the open - /// input slot remembered for the next frame in. - fn pump(&mut self, bytes: &mut Vec, key: &mut bool) -> Result<(), String> { + fn event(&mut self) -> Result, String> { loop { // SAFETY: the event generator belongs to this encoder and is not held // elsewhere; each event is this call's own, read once and dropped. let event = match unsafe { self.events.GetEvent(MF_EVENT_FLAG_NO_WAIT) } { Ok(event) => event, - Err(err) if err.code() == MF_E_NO_EVENTS_AVAILABLE => return Ok(()), + Err(err) if err.code() == MF_E_NO_EVENTS_AVAILABLE => return Ok(None), Err(err) => return Err(format!("the hardware encoder's events failed: {err}")), }; let kind = unsafe { event.GetType() }.map_err(|err| format!("the hardware encoder's events could not be read: {err}"))?; if kind == METransformHaveOutput.0 as u32 { - self.take_output(bytes, key)?; - } else if kind == METransformNeedInput.0 as u32 { - self.wants_input = true; + return Ok(Some(Event::HaveOutput)); + } + if kind == METransformNeedInput.0 as u32 { + return Ok(Some(Event::NeedInput)); } // METransformDrainComplete and METransformInputStreamStateChanged need nothing // from a share: nothing is ever drained, and the input state is read back @@ -363,9 +441,28 @@ impl Encoder { } } - /// One encoded frame out of the transform, its bytes appended and its keyframe-ness - /// remembered. - fn take_output(&mut self, bytes: &mut Vec, key: &mut bool) -> Result<(), String> { + fn give(&mut self, planes: &Planes, stride: usize, frame: usize, at_100ns: i64, lasting_100ns: i64) -> Result<(), String> { + // SAFETY: the sample is created here, filled here, and handed to the transform, + // which owns its reference from `ProcessInput` on; the buffer pointer is used + // within its lock below and gone before the lock is let go. + unsafe { + let buffer = MFCreateMemoryBuffer(frame as u32).map_err(|err| format!("the frame could not be given room: {err}"))?; + let mut data: *mut u8 = null_mut(); + buffer.Lock(&mut data, None, None).map_err(|err| format!("the frame could not be written: {err}"))?; + nv12(planes, stride, std::slice::from_raw_parts_mut(data, frame)); + let _ = buffer.SetCurrentLength(frame as u32); + buffer.Unlock().map_err(|err| format!("the frame could not be handed over: {err}"))?; + + let sample = MFCreateSample().map_err(|err| format!("the frame could not be wrapped: {err}"))?; + sample.AddBuffer(&buffer).map_err(|err| format!("the frame could not be wrapped: {err}"))?; + sample.SetSampleTime(at_100ns).map_err(|err| format!("the frame's time was refused: {err}"))?; + sample.SetSampleDuration(lasting_100ns.max(1)).map_err(|err| format!("the frame's length was refused: {err}"))?; + self.transform.ProcessInput(0, &sample, 0).map_err(|err| format!("the hardware encoder would not take the frame: {err}"))?; + Ok(()) + } + } + + fn take(&mut self) -> Result, String> { // SAFETY: the output slot belongs to this call; the sample the transform fills is // taken out of it and read before it is dropped here. unsafe { @@ -378,25 +475,50 @@ impl Encoder { let mut status = 0u32; self.transform.ProcessOutput(0, &mut slot, &mut status).map_err(|err| format!("the hardware encoder gave no frame: {err}"))?; let Some(sample) = std::mem::ManuallyDrop::take(&mut slot[0].pSample) else { - return Ok(()); + return Ok(None); }; - if sample.GetUINT32(&MFSampleExtension_CleanPoint).is_ok() { - *key = true; - } + // The attribute's value is what a keyframe is, not its existence: a delta + // frame carries the extension set to zero. + let clean = sample.GetUINT32(&MFSampleExtension_CleanPoint).ok(); // The sample's buffers are made one here, whatever count the encoder used. let contiguous = sample.ConvertToContiguousBuffer().map_err(|err| format!("the encoded frame could not be read: {err}"))?; let mut data: *mut u8 = null_mut(); let mut length = 0u32; contiguous.Lock(&mut data, None, Some(&mut length)).map_err(|err| format!("the encoded frame could not be read: {err}"))?; - bytes.extend_from_slice(&annex_b(std::slice::from_raw_parts(data, length as usize))); + let bytes = annex_b(std::slice::from_raw_parts(data, length as usize)); contiguous.Unlock().map_err(|err| format!("the encoded frame could not be let go: {err}"))?; - Ok(()) + Ok(Some(Taken { bytes, clean })) } } -} -impl Drop for Encoder { - fn drop(&mut self) { + fn tune(&mut self, bps: u32) -> bool { + let Some(codec) = &self.codec else { + return false; + }; + // SAFETY: the property is read through the codec handle this encoder holds, and + // the variant outlives the call. + match unsafe { codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(bps)) } { + Ok(()) => true, + Err(err) => { + tracing::warn!("mediad: the hardware encoder refused a bitrate of {}: {err}", bps); + false + } + } + } + + fn force_key(&mut self) { + let Some(codec) = &self.codec else { + return; + }; + // The property carries no value; setting it is the ask. An encoder without codec + // properties keeps its own keyframe pace. + // SAFETY: as in `tune`, with a property whose value is ignored. + if let Err(err) = unsafe { codec.SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &counted(0)) } { + tracing::debug!("mediad: the hardware encoder refused a keyframe: {err}"); + } + } + + fn end(&mut self) { // SAFETY: the transform is still owned here; both messages are its own. unsafe { let _ = self.transform.ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); @@ -405,6 +527,151 @@ impl Drop for Encoder { } } +/// The machine's hardware H.264 encoder, held as the async MFT behind the [`Mft`] trait, +/// the frame's shape, and what the track asked for. It is opened, driven, and dropped on +/// one thread (the share's or the camera's capture thread); COM and Media Foundation live +/// in the [`Platform`] guard, let go after the transform is. +pub struct Encoder { + mft: Box, + width: usize, + height: usize, + /// The stride promised in the input type: the width rounded up as the hardware wants. + stride: usize, + /// One NV12 frame at that stride: the Y plane and the two interleaved chroma planes. + frame: usize, + fps: f32, + /// What the encoder holds now, so a tune is only sent when the ask moved. + bps: u32, + /// Whether the encoder has an input slot open: a `METransformNeedInput` seen and not + /// yet fed. + wants_input: bool, + /// Whether a frame has gone in yet: the first may wait the encoder's warmup for its + /// bits, every later one waits at most [`OUTPUT_WAIT`]. + fed: bool, + /// COM and Media Foundation, held so they outlive the transform and are let go after + /// it (the field after the transform's own). + _platform: Platform, +} + +impl Encoder { + /// The first hardware encoder that takes NV12 and answers H.264, set up for CBR at + /// `bps` and at most `fps`, level 5.1 for a screen and 3.1 for a camera, at the + /// constrained baseline the SFU takes. Its own words when there is no such encoder, or + /// one would not take its setup, or one would put a profile other than the baseline on + /// the wire; the caller falls back to OpenH264 on any of them. + pub fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + Self::open_with(&Hardware, platform()?, width, height, bps, fps, screen) + } + + /// The open, against any source of transforms. The output type is negotiated first, at + /// the promised profile — the machine's encoders refuse an input type while the output + /// type is unset — and what the transform kept is held to the promise before the input + /// type, the codec properties and streaming follow. Nothing is retried without the + /// profile: whatever else the driver picked would go out on the wire as a stream the + /// SFU and the receivers were told never to expect, so such a driver is refused and + /// OpenH264 takes the job. + fn open_with(source: &dyn Source, platform: Platform, width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + let (width, height) = (width.max(2) as usize, height.max(2) as usize); + let stride = (width + STRIDE_ALIGN - 1) & !(STRIDE_ALIGN - 1); + let frame = stride * height * 3 / 2; + + let mut mft = source.activate()?; + mft.unlock()?; + let kept = mft.set_output_type(width, height, bps, fps, screen)? + .ok_or_else(|| "the hardware encoder would not say which H.264 profile it encodes.".to_owned())?; + if kept != BASELINE && kept != CONSTRAINED_BASELINE { + return Err(format!("the hardware encoder encodes H.264 profile {kept}, not the baseline the SFU takes.")); + } + mft.set_input_type(width, height, stride, frame, fps)?; + mft.configure(bps); + mft.begin()?; + let mut encoder = Encoder { mft, width, height, stride, frame, fps, bps, wants_input: false, fed: false, _platform: platform }; + // The first input slot is usually open before the first frame; wait a moment + // for it, so the first frame of a share is fed at once and not dropped. + let mut nothing = Vec::new(); + let until = Instant::now() + ANSWER_WAIT; + while !encoder.wants_input && Instant::now() < until { + encoder.drain(&mut nothing)?; + if !encoder.wants_input { + std::thread::sleep(POLL); + } + } + Ok(encoder) + } + + /// One frame in, whatever is out. A frame whose bits are still on their way comes back + /// empty, like a skip; an encoded frame comes back marked when it is a keyframe, so + /// the share sends it as one. After the first frame is in, the wait for a frame's own + /// bits is bounded well under a frame's time, so the capture thread is never held + /// long. + pub fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { + if planes.width != self.width || planes.height != self.height { + return Err(format!("The encoder was opened for {}×{} and handed {}×{}.", self.width, self.height, planes.width, planes.height)); + } + if ask.bps != self.bps && ask.bps > 0 && self.mft.tune(ask.bps) { + self.bps = ask.bps; + } + if ask.key { + self.mft.force_key(); + } + // What is already out, before this frame goes in: the machine may be a frame ahead. + let mut bytes = Vec::new(); + let mut key = self.drain(&mut bytes)?; + if self.wants_input { + // The ask's clock is milliseconds; Media Foundation's is 100 nanoseconds. + let at = (ask.at.as_millis() as i64).saturating_mul(10_000); + let lasting = (10_000_000.0 / self.fps.max(1.0)) as i64; + self.mft.give(planes, self.stride, self.frame, at, lasting)?; + self.wants_input = false; + let first = !self.fed; + self.fed = true; + // A bounded moment for this frame's own bits. The first frame may wait the + // encoder's warmup; every later one waits no longer than [`OUTPUT_WAIT`], so a + // driver holding frames back never stalls the capture thread. When the bits + // are not out in time, the empty frame goes and the pump catches them next + // time. + let until = Instant::now() + if first { ANSWER_WAIT } else { OUTPUT_WAIT }; + while bytes.is_empty() && Instant::now() < until { + key |= self.drain(&mut bytes)?; + if bytes.is_empty() { + std::thread::sleep(POLL); + } + } + } + if bytes.is_empty() { + key = false; + } + Ok(Encoded { bytes, key }) + } + + /// Everything the event queue holds now, without waiting: frames out, their bytes + /// appended, and whether what came out holds a keyframe; the open input slot is + /// remembered for the next frame in. + fn drain(&mut self, bytes: &mut Vec) -> Result { + let mut key = false; + while let Some(event) = self.mft.event()? { + match event { + Event::HaveOutput => { + if let Some(taken) = self.mft.take()? { + if clean_point_is_key(taken.clean) { + key = true; + } + bytes.extend_from_slice(&taken.bytes); + } + } + Event::NeedInput => self.wants_input = true, + } + } + Ok(key) + } +} + +impl Drop for Encoder { + fn drop(&mut self) { + self.mft.end(); + } +} + /// The input type the encoder is fed: NV12 at the promised stride, whole frames, at the /// size and rate the track asked for. fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result { @@ -425,9 +692,11 @@ fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32 } /// The output type asked for: H.264 at the bitrate, size and rate of the track, at the -/// profile the SFU takes when `with_profile` says so. An encoder that keeps none of them -/// is asked again without. -fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool, with_profile: bool) -> Result { +/// constrained baseline the SFU takes. What the encoder actually kept is read back and +/// held to the promise by [`Encoder::open_with`]; there is no retry without the profile, +/// because whatever else the driver picked would go out on the wire as a stream the SFU +/// was told never to expect. A level holds the size and rate, 5.1 for a 1440p share at 60. +fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result { let level = if screen { eAVEncH264VLevel5_1 } else { eAVEncH264VLevel3_1 }; // SAFETY: as in `input_type`, the type is returned to be set. unsafe { @@ -439,12 +708,8 @@ fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool, wi kind.SetUINT64(&MF_MT_FRAME_RATE, rate(fps)).map_err(|err| format!("the output type could not be built: {err}"))?; kind.SetUINT32(&MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; kind.SetUINT64(&MF_MT_PIXEL_ASPECT_RATIO, size(1, 1)).map_err(|err| format!("the output type could not be built: {err}"))?; - if with_profile { - // Constrained baseline is what the SFU takes, and what OpenH264 sends; a level - // holds the size and rate, 5.1 for a 1440p share at 60. - kind.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_ConstrainedBase.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; - kind.SetUINT32(&MF_MT_MPEG2_LEVEL, level.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; - } + kind.SetUINT32(&MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_ConstrainedBase.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; + kind.SetUINT32(&MF_MT_MPEG2_LEVEL, level.0 as u32).map_err(|err| format!("the output type could not be built: {err}"))?; Ok(kind) } } @@ -541,7 +806,12 @@ fn annex_b(bytes: &[u8]) -> Vec { #[cfg(test)] mod tests { + use std::cell::RefCell; + use std::collections::VecDeque; + use std::rc::Rc; + use super::*; + use windows::Win32::Media::MediaFoundation::eAVEncH264VProfile_Main; #[test] fn a_clean_probe_means_hardware_and_any_failure_means_the_download() { @@ -614,4 +884,307 @@ mod tests { assert_eq!(rate(29.97), (29_970u64 << 32) | 1000); assert_eq!(rate(0.), 1u64 << 32 | 1000); } + + /// A transform with no machine behind it: it answers as its fields say and keeps what + /// it is handed, so the open, the encode and the keyframe flag are held against the + /// same words the hardware answers, on any machine. + #[derive(Default)] + struct FakeMft { + /// What `set_output_type` answers instead of setting a type. + output_refusal: Option, + /// What `set_input_type` answers instead of setting a type. + input_refusal: Option, + /// The profile the kept type names: `None` is a type that names none. + negotiated_profile: Option, + /// Events waiting to be read. + events: VecDeque, + /// Frames waiting to be taken. + outputs: VecDeque, + /// What comes out after a frame goes in, for the frame-loop tests. + after_give: Option, + /// The frames handed in, as (width, height). + given: Vec<(usize, usize)>, + /// The bitrates the tunes asked for. + tuned: Vec, + /// The bitrate the codec properties settled at open. + configured: Option, + keys_forced: u32, + unlocked: bool, + output_set: bool, + input_set: bool, + began: bool, + ended: bool, + } + + impl Mft for FakeMft { + fn unlock(&mut self) -> Result<(), String> { + self.unlocked = true; + Ok(()) + } + + fn set_output_type(&mut self, _width: usize, _height: usize, _bps: u32, _fps: f32, _screen: bool) -> Result, String> { + if let Some(why) = self.output_refusal.clone() { + return Err(why); + } + self.output_set = true; + Ok(self.negotiated_profile) + } + + fn set_input_type(&mut self, _width: usize, _height: usize, _stride: usize, _frame: usize, _fps: f32) -> Result<(), String> { + if let Some(why) = self.input_refusal.clone() { + return Err(why); + } + self.input_set = true; + Ok(()) + } + + fn configure(&mut self, bps: u32) { + self.configured = Some(bps); + } + + fn begin(&mut self) -> Result<(), String> { + self.began = true; + Ok(()) + } + + fn event(&mut self) -> Result, String> { + Ok(self.events.pop_front()) + } + + fn give(&mut self, planes: &Planes, _stride: usize, _frame: usize, _at_100ns: i64, _lasting_100ns: i64) -> Result<(), String> { + self.given.push((planes.width, planes.height)); + // The input slot reopens behind the frame taken in, the way the machine's + // encoders answer. + self.events.push_back(Event::NeedInput); + if let Some(taken) = self.after_give.clone() { + self.events.push_back(Event::HaveOutput); + self.outputs.push_back(taken); + } + Ok(()) + } + + fn take(&mut self) -> Result, String> { + Ok(self.outputs.pop_front()) + } + + fn tune(&mut self, bps: u32) -> bool { + self.tuned.push(bps); + true + } + + fn force_key(&mut self) { + self.keys_forced += 1; + } + + fn end(&mut self) { + self.ended = true; + } + } + + /// The fake the test keeps a handle into: every ask of the encoder lands in the one + /// fake both sides see. + #[derive(Clone)] + struct SharedFake(Rc>); + + impl SharedFake { + fn new(fake: FakeMft) -> SharedFake { + SharedFake(Rc::new(RefCell::new(fake))) + } + + fn with(&self, read: impl FnOnce(&FakeMft) -> R) -> R { + read(&self.0.borrow()) + } + } + + impl Mft for SharedFake { + fn unlock(&mut self) -> Result<(), String> { + self.0.borrow_mut().unlock() + } + + fn set_output_type(&mut self, width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result, String> { + self.0.borrow_mut().set_output_type(width, height, bps, fps, screen) + } + + fn set_input_type(&mut self, width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result<(), String> { + self.0.borrow_mut().set_input_type(width, height, stride, frame, fps) + } + + fn configure(&mut self, bps: u32) { + self.0.borrow_mut().configure(bps) + } + + fn begin(&mut self) -> Result<(), String> { + self.0.borrow_mut().begin() + } + + fn event(&mut self) -> Result, String> { + self.0.borrow_mut().event() + } + + fn give(&mut self, planes: &Planes, stride: usize, frame: usize, at_100ns: i64, lasting_100ns: i64) -> Result<(), String> { + self.0.borrow_mut().give(planes, stride, frame, at_100ns, lasting_100ns) + } + + fn take(&mut self) -> Result, String> { + self.0.borrow_mut().take() + } + + fn tune(&mut self, bps: u32) -> bool { + self.0.borrow_mut().tune(bps) + } + + fn force_key(&mut self) { + self.0.borrow_mut().force_key() + } + + fn end(&mut self) { + self.0.borrow_mut().end() + } + } + + // SAFETY: a test's fake lives and dies on the one thread the test runs on, and the + // encoder is opened, driven and dropped on that same thread. + unsafe impl Send for SharedFake {} + + /// One fake transform, handed over when the open asks for it. + struct FakeSource(SharedFake); + + impl Source for FakeSource { + fn activate(&self) -> Result, String> { + Ok(Box::new(self.0.clone())) + } + } + + /// A 4×4 frame the way the share's picture arrives. + fn planes() -> crate::picture::Planes { + let rgba = vec![255u8; 4 * 4 * 4]; + crate::picture::i420(&rgba, 4, 4, 4, 4, false) + } + + /// An open against a fake that keeps the constrained baseline, with its first input + /// slot already open. + fn opened(shared: SharedFake) -> Encoder { + Encoder::open_with(&FakeSource(shared), Platform::unstarted(), 4, 4, 2_500_000, 30., true).expect("the fake encoder opens") + } + + fn baseline_fake() -> FakeMft { + let mut fake = FakeMft::default(); + fake.negotiated_profile = Some(BASELINE); + fake.events.push_back(Event::NeedInput); + fake + } + + /// An MFT that refuses the H.264 output type fails the Media Foundation encoder in its + /// own words, and `choose` sends every such refusal to OpenH264 — the fallback a + /// failed probe already takes. Nothing is retried without the profile. + #[test] + fn an_encoder_that_refuses_the_output_type_is_failed_in_its_own_words() { + let mut fake = FakeMft::default(); + fake.output_refusal = Some("would not take an H.264 output".to_owned()); + let shared = SharedFake::new(fake); + let why = Encoder::open_with(&FakeSource(shared.clone()), Platform::unstarted(), 4, 4, 2_500_000, 30., true) + .expect_err("a refused output type is a refused encoder"); + assert!(why.contains("H.264 output"), "{why}"); + assert!(!shared.with(|fake| fake.input_set), "the NV12 input ask never came after the refusal"); + // And the refusal is the fallback's cue, the same as a failed probe. + assert_eq!(choose(&Err(why)), Choice::OpenH264); + } + + /// A driver that keeps Main or High, or names no profile at all, would quietly change + /// what goes on the wire; both are refused in words that say why, and the baselines + /// the SFU was promised are the ones that open. + #[test] + fn a_profile_other_than_the_baseline_promised_is_refused() { + let mut high = FakeMft::default(); + high.negotiated_profile = Some(eAVEncH264VProfile_Main.0 as u32); + let why = Encoder::open_with(&FakeSource(SharedFake::new(high)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) + .expect_err("Main is not the baseline the SFU takes"); + assert!(why.contains("not the baseline"), "{why}"); + assert_eq!(choose(&Err(why)), Choice::OpenH264); + + let mut silent = FakeMft::default(); + silent.negotiated_profile = None; + let why = Encoder::open_with(&FakeSource(SharedFake::new(silent)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) + .expect_err("a kept type that names no profile cannot be held to the promise"); + assert!(why.contains("profile"), "{why}"); + + // Constrained baseline and plain baseline are the promise kept. + for kept in [CONSTRAINED_BASELINE, BASELINE] { + let mut fake = FakeMft::default(); + fake.negotiated_profile = Some(kept); + fake.events.push_back(Event::NeedInput); + drop(opened(SharedFake::new(fake))); + } + } + + /// `MFSampleExtension_CleanPoint` says keyframe only by its value: a delta frame that + /// carries the extension as zero, and a sample that does not carry it at all, are not + /// keyframes, whatever a mere "the attribute is there" would have said. + #[test] + fn a_frame_is_a_keyframe_only_when_the_sample_says_it_is_a_clean_point() { + assert!(!clean_point_is_key(None)); + assert!(!clean_point_is_key(Some(0))); + assert!(clean_point_is_key(Some(1))); + for (clean, key) in [(Some(1u32), true), (Some(0u32), false), (None, false)] { + let mut fake = baseline_fake(); + fake.after_give = Some(Taken { bytes: vec![0, 0, 0, 1, 0x65], clean }); + let mut encoder = opened(SharedFake::new(fake)); + let encoded = encoder.encode(&planes(), &Ask::default()).expect("the fake encoder answers"); + assert_eq!(encoded.key, key, "clean point {clean:?}"); + assert!(!encoded.bytes.is_empty(), "the frame's bits came out"); + } + } + + /// An open encoder takes frames: the frames go in at the size it was opened for, the + /// keyframe ask reaches the codec, the codec properties settle at open, and a tune is + /// only sent when the ask moves. + #[test] + fn an_open_encoder_takes_frames_and_holds_the_asks() { + let mut fake = baseline_fake(); + fake.after_give = Some(Taken { bytes: vec![0, 0, 0, 1, 0x65], clean: Some(1) }); + let shared = SharedFake::new(fake); + let mut encoder = opened(shared.clone()); + + let asked = Ask { key: true, ..Ask::default() }; + let out = encoder.encode(&planes(), &asked).expect("the first frame answers"); + assert!(out.key, "the first output is a clean point"); + shared.with(|fake| { + assert!(fake.unlocked && fake.output_set && fake.input_set && fake.began, "the open ran whole"); + assert_eq!(fake.configured, Some(2_500_000), "the codec properties settled at open"); + assert_eq!(fake.given, vec![(4, 4)], "the frame went in"); + assert_eq!(fake.keys_forced, 1, "the keyframe ask reached the codec"); + assert!(fake.tuned.is_empty(), "the ask did not move, so no tune went out"); + }); + + // The same rate again: no tune. A moved rate: one tune, and the encoder keeps it. + encoder.encode(&planes(), &Ask::default()).expect("the second frame answers"); + let moved = Ask { bps: 1_000_000, ..Ask::default() }; + encoder.encode(&planes(), &moved).expect("the third frame answers"); + shared.with(|fake| { + assert_eq!(fake.tuned, vec![1_000_000], "only the moved ask is tuned"); + assert_eq!(fake.keys_forced, 1, "no keyframe ask without one"); + assert_eq!(fake.given.len(), 3, "every frame with an open slot went in"); + }); + } + + /// A close is a flush and an end of streaming, said to the transform before it goes. + #[test] + fn a_closed_encoder_ends_its_stream() { + let shared = SharedFake::new(baseline_fake()); + drop(opened(shared.clone())); + shared.with(|fake| assert!(fake.ended, "the close reached the transform")); + } + + /// The probe answers on a thread of its own, so the machine's words are taken on a + /// fresh apartment and the asking thread's is never asked. Here it is held against + /// running whole: it answers, once, and its words are its own either way. + #[test] + fn the_probe_answers_on_a_thread_of_its_own() { + let answer = probe_thread().expect("the probe's thread starts"); + // Whatever this machine answers, the probe ran whole: COM came up and went, and + // Media Foundation with it. A machine with no hardware MFT is answered in words. + if let Err(why) = answer { + assert!(!why.is_empty(), "a refusal says why"); + } + } } diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index 9472d644..1dd4dbd2 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -48,6 +48,18 @@ struct Coded { backend: Backend, width: u32, height: u32, + /// What the track asked at open: what a mid-share failover sets the software encoder + /// up with. Windows only: the failover is. + #[cfg(windows)] + bps: u32, + #[cfg(windows)] + fps: f32, + #[cfg(windows)] + screen: bool, + /// Whether the hardware's mid-share failure has been said once, so a failing frame a + /// second does not say it a second a time. + #[cfg(windows)] + said: bool, } /// One of the two encoders behind [`Coded`]: the machine's own hardware, or Cisco's @@ -137,22 +149,86 @@ impl Coded { /// OpenH264, downloaded at run time, takes every other machine and every refusal, with /// the words it has always had. fn open(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { - #[cfg(windows)] + Ok(Coded::with(Self::backend(width, height, bps, fps, screen)?, width, height, bps, fps, screen)) + } + + #[cfg(windows)] + fn with(backend: Backend, width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Coded { + Coded { backend, width, height, bps, fps, screen, said: false } + } + + #[cfg(not(windows))] + fn with(backend: Backend, width: u32, height: u32, _bps: u32, _fps: f32, _screen: bool) -> Coded { + Coded { backend, width, height } + } + + /// The machine's own hardware when Windows lists one, else OpenH264. The one place a + /// refusal of the hardware becomes the software encoder. + #[cfg(windows)] + fn backend(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { if mf_encoder::choose(&mf_encoder::available()) == mf_encoder::Choice::MediaFoundation { - match mf_encoder::Encoder::open(width, height, bps, fps, screen) { - Ok(encoder) => return Ok(Coded { backend: Backend::MediaFoundation(encoder), width, height }), - Err(why) => tracing::warn!("mediad: the Media Foundation encoder could not be set up ({why}); OpenH264 takes over."), - } + return settle(mf_encoder::Encoder::open(width, height, bps, fps, screen), || Backend::open_h264(bps, fps, screen)); } - let backend = Backend::open_h264(bps, fps, screen)?; - Ok(Coded { backend, width, height }) + Backend::open_h264(bps, fps, screen) + } + + #[cfg(not(windows))] + fn backend(width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Result { + Backend::open_h264(bps, fps, screen) } fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { + // The hardware failing mid-share does not end the share: one frame's error swaps + // the backend to OpenH264, whose first frame is a keyframe, and the track keeps + // going. The swap itself may have to wait for the OpenH264 download; until it + // lands the frame is skipped and the next one tries again. + #[cfg(windows)] + if matches!(self.backend, Backend::MediaFoundation(_)) { + match self.backend.encode(planes, ask) { + Ok(encoded) => return Ok(encoded), + Err(why) => { + if !self.said { + tracing::warn!("mediad: the Media Foundation encoder failed mid-share ({why}); OpenH264 takes over with a keyframe."); + self.said = true; + } else { + tracing::debug!("mediad: the Media Foundation encoder failed again ({why})."); + } + match Backend::open_h264(self.bps, self.fps, self.screen) { + Ok(fresh) => { + self.backend = fresh; + // The fresh encoder's first frame is a keyframe of its own; the + // ask is forced besides, so one that was due is not lost to the + // swap. + let forced = Ask { key: true, ..*ask }; + return self.backend.encode(planes, &forced); + } + Err(why) => { + tracing::debug!("mediad: OpenH264 is not ready to take over ({why}); this frame is skipped."); + return Ok(Encoded::default()); + } + } + } + } + } self.backend.encode(planes, ask) } } +/// The encoder the hardware gave, else the software one, with the refusal's words said in +/// the log. The one place an open of the hardware becomes the fallback, whatever the +/// refusal was: no encoder, a refused type, or a driver that would put a profile other +/// than the baseline on the wire. +#[cfg(windows)] +fn settle(first: Result, software: impl FnOnce() -> Result) -> Result { + match first { + Ok(encoder) => Ok(Backend::MediaFoundation(encoder)), + Err(why) => { + tracing::warn!("mediad: the Media Foundation encoder could not be set up ({why}); OpenH264 takes over."); + software() + } + } +} + /// Whether the OpenH264 binary is wanted before this track's first frame: on Windows the /// machine's own hardware may take the job, and then no download is needed. A decode still /// wants the binary whatever the send side chose, so `warm` runs on regardless. @@ -997,6 +1073,19 @@ pub fn probe_camera() -> Result<(), String> { mod tests { use super::*; + /// A refusal of the hardware settles on the software encoder, and the software + /// encoder's own answer is what the track gets: the hardware's words are said in the + /// log, and what comes back from the fallback is passed on whole, a refusal included. + /// The hardware opening is the one-line arm above it. + #[cfg(windows)] + #[test] + fn a_refused_hardware_encoder_settles_on_the_software_one() { + let refused = Err("the hardware encoder encodes H.264 profile 100, not the baseline the SFU takes.".to_owned()); + let why = settle(refused, || Err("OpenH264 could not be downloaded. The call stays a voice call.".to_owned())) + .expect_err("the software answer is the track's"); + assert!(why.contains("OpenH264"), "{why}"); + } + #[test] fn openh264_is_fetched_from_cisco() { let name = library_name().unwrap_or("openh264-2.6.0-win64.dll"); diff --git a/e2e/scenarios/hardware-encoder-fallback/README.md b/e2e/scenarios/hardware-encoder-fallback/README.md index 3d3eaa7c..1669d140 100644 --- a/e2e/scenarios/hardware-encoder-fallback/README.md +++ b/e2e/scenarios/hardware-encoder-fallback/README.md @@ -1,13 +1,22 @@ # Hardware encoder fallback -Run `sh e2e/scenarios/hardware-encoder-fallback/run.sh`. It needs no devices and no -network: the scenario drives the selection logic and the pure NV12/Annex B helpers of -`crates/mediad/src/mf_encoder.rs`. +Run `sh e2e/scenarios/hardware-encoder-fallback/run.sh`. It needs no devices, no encoder +hardware and no network: the scenario drives the tests of +`crates/mediad/src/mf_encoder.rs` and the fallback seam in `picture/desktop.rs` — the +encoder selection, the fake-MFT seam that holds the open (the H.264 output type refused, +a driver that kept Main/High or named no profile, a keyframe that only counts when the +sample says `MFSampleExtension_CleanPoint`, the frame loop with its asks), the pure NV12 +and Annex B helpers, and the place a refusal of the hardware becomes the OpenH264 +fallback. On the base the scenario fails: the module does not exist, so cargo reports no matching -tests. With the branch the Media Foundation hardware encoder is tried first (when the -machine lists one that takes NV12 and answers H.264) and the OpenH264 runtime download -stays the fallback with its own covered path. +tests. With the branch the Media Foundation hardware encoder is tried first (the output +type negotiated first, at the constrained baseline, and what the driver kept read back +and held to that), and the OpenH264 runtime download stays the fallback with its own +covered path. The on-hardware record (the lab driving a real share over a call) needs a machine with a -hardware MFT; that recording is the released-client proof this scenario points to. +hardware MFT; that recording is the released-client proof this scenario points to. What +this scenario can say without that machine: a driver that would not keep the promised +wire is refused in its own words and OpenH264 takes the job, and nothing else changed +about what goes out. diff --git a/e2e/scenarios/hardware-encoder-fallback/run.sh b/e2e/scenarios/hardware-encoder-fallback/run.sh index 27cdf200..583bb332 100644 --- a/e2e/scenarios/hardware-encoder-fallback/run.sh +++ b/e2e/scenarios/hardware-encoder-fallback/run.sh @@ -1,6 +1,9 @@ #!/bin/sh -# The Media Foundation hardware encoder's selection and helpers, unit-tested on Windows (the -# module is cfg(windows), so this driver runs where it compiles). No network, no devices. +# The Media Foundation hardware encoder's tests, unit-tested on Windows (the module is +# cfg(windows), so this driver runs where it compiles): the selection logic, the fake-MFT +# seam that holds the open, the profile refusal, the frame loop and the keyframe flag +# without hardware, and the pure NV12/Annex B helpers. No network, no devices, no encoder +# hardware. set -eu cd "$(dirname "$0")/../../.." log=$(mktemp) @@ -8,23 +11,30 @@ trap 'rm -f "$log"' EXIT HUP INT TERM run() { name=$1 printf '%s\n' "media foundation encoder: $name" - if ! cargo test -p mediad --lib "mf_encoder::tests::$name" -- --exact --nocapture --test-threads=1 >"$log" 2>&1; then + if ! cargo test -p mediad --lib "$name" -- --exact --nocapture --test-threads=1 >"$log" 2>&1; then cat "$log" return 1 fi cat "$log" grep -Fq "running 1 test" "$log" - grep -Fq "test mf_encoder::tests::$name ... ok" "$log" + grep -Fq "test $name ... ok" "$log" if grep -Eq '(^|[[:space:]])skipped:' "$log"; then echo 'FAIL: a fixture skipped' >&2 return 1 fi } -run a_clean_probe_means_hardware_and_any_failure_means_the_download -run nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved -run nv12_of_a_short_buffer_writes_nothing_past_it -run annex_b_already_annex_b_is_untouched -run length_prefixed_units_come_out_behind_start_codes -run a_truncated_or_empty_stream_never_runs_past_itself -run sizes_and_rates_are_packed_width_over_height_and_thousandths -printf '%s\n' 'PASS: the hardware encoder is chosen when present and OpenH264 stays the fallback.' +run mf_encoder::tests::a_clean_probe_means_hardware_and_any_failure_means_the_download +run mf_encoder::tests::an_encoder_that_refuses_the_output_type_is_failed_in_its_own_words +run mf_encoder::tests::a_profile_other_than_the_baseline_promised_is_refused +run mf_encoder::tests::a_frame_is_a_keyframe_only_when_the_sample_says_it_is_a_clean_point +run mf_encoder::tests::an_open_encoder_takes_frames_and_holds_the_asks +run mf_encoder::tests::a_closed_encoder_ends_its_stream +run mf_encoder::tests::the_probe_answers_on_a_thread_of_its_own +run mf_encoder::tests::nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved +run mf_encoder::tests::nv12_of_a_short_buffer_writes_nothing_past_it +run mf_encoder::tests::annex_b_already_annex_b_is_untouched +run mf_encoder::tests::length_prefixed_units_come_out_behind_start_codes +run mf_encoder::tests::a_truncated_or_empty_stream_never_runs_past_itself +run mf_encoder::tests::sizes_and_rates_are_packed_width_over_height_and_thousandths +run picture::desktop::tests::a_refused_hardware_encoder_settles_on_the_software_one +printf '%s\n' 'PASS: the hardware encoder is chosen when present, refuses a wire it was not promised, and OpenH264 stays the fallback.' diff --git a/e2e/scenarios/hardware-encoder-fallback/scenario.json b/e2e/scenarios/hardware-encoder-fallback/scenario.json index e4b07a2e..4817620b 100644 --- a/e2e/scenarios/hardware-encoder-fallback/scenario.json +++ b/e2e/scenarios/hardware-encoder-fallback/scenario.json @@ -5,6 +5,6 @@ "needs": [], "release": false, "driver": "run.sh", - "proof_scope": "encoder selection and helpers, unit-level; the on-hardware record needs a machine with a hardware MFT", + "proof_scope": "encoder selection, the fake-MFT open/refusal/keyframe path, and the NV12/Annex B helpers, unit-level; the on-hardware record needs a machine with a hardware MFT", "native_client_proof": "the released client shows screen share and camera running on the machine's own hardware encoder, with the OpenH264 download as the fallback" } From 6913474924de734f9063f1aba22297918aa34495 Mon Sep 17 00:00:00 2001 From: unRekable Date: Wed, 11 Nov 2026 00:51:24 +0100 Subject: [PATCH 6/7] The tests hold their refusals with let-else, and the casts come without a block they do not need --- crates/mediad/src/mf_encoder.rs | 26 +++++++++++++++----------- crates/mediad/src/picture/desktop.rs | 5 +++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs index 12781606..bf526943 100644 --- a/crates/mediad/src/mf_encoder.rs +++ b/crates/mediad/src/mf_encoder.rs @@ -323,12 +323,13 @@ impl Source for Hardware { // SAFETY: the activate is still owned here. unsafe { let _ = activate.ShutdownObject(); } drop(activate); - // SAFETY: both casts are on the transform this open created. - let events: IMFMediaEventGenerator = unsafe { transform.cast() } + // Both casts are on the transform this open created; `cast` is safe in + // windows-core: it cannot name a null or wrong-vtable object without an `Err`. + let events: IMFMediaEventGenerator = transform.cast() .map_err(|err| format!("the hardware encoder has no event queue: {err}"))?; // `codec` is `None` when the MFT exposes no codec properties, which leaves a // keyframe ask and a bitrate move without their handle; the picture still encodes. - let codec: Option = unsafe { transform.cast() }.ok(); + let codec: Option = transform.cast().ok(); Ok(Box::new(RealMft { transform, events, codec })) } } @@ -674,7 +675,7 @@ impl Drop for Encoder { /// The input type the encoder is fed: NV12 at the promised stride, whole frames, at the /// size and rate the track asked for. -fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result { +fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32) -> Result { // SAFETY: the media type is created here and returned; the caller sets it before drop. unsafe { let kind = MFCreateMediaType().map_err(|err| format!("the input type could not be built: {err}"))?; @@ -696,7 +697,7 @@ fn input_type(width: usize, height: usize, stride: usize, frame: usize, fps: f32 /// held to the promise by [`Encoder::open_with`]; there is no retry without the profile, /// because whatever else the driver picked would go out on the wire as a stream the SFU /// was told never to expect. A level holds the size and rate, 5.1 for a 1440p share at 60. -fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result { +fn output_type(width: usize, height: usize, bps: u32, fps: f32, screen: bool) -> Result { let level = if screen { eAVEncH264VLevel5_1 } else { eAVEncH264VLevel3_1 }; // SAFETY: as in `input_type`, the type is returned to be set. unsafe { @@ -1082,8 +1083,9 @@ mod tests { let mut fake = FakeMft::default(); fake.output_refusal = Some("would not take an H.264 output".to_owned()); let shared = SharedFake::new(fake); - let why = Encoder::open_with(&FakeSource(shared.clone()), Platform::unstarted(), 4, 4, 2_500_000, 30., true) - .expect_err("a refused output type is a refused encoder"); + let Err(why) = Encoder::open_with(&FakeSource(shared.clone()), Platform::unstarted(), 4, 4, 2_500_000, 30., true) else { + panic!("a refused output type is a refused encoder"); + }; assert!(why.contains("H.264 output"), "{why}"); assert!(!shared.with(|fake| fake.input_set), "the NV12 input ask never came after the refusal"); // And the refusal is the fallback's cue, the same as a failed probe. @@ -1097,15 +1099,17 @@ mod tests { fn a_profile_other_than_the_baseline_promised_is_refused() { let mut high = FakeMft::default(); high.negotiated_profile = Some(eAVEncH264VProfile_Main.0 as u32); - let why = Encoder::open_with(&FakeSource(SharedFake::new(high)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) - .expect_err("Main is not the baseline the SFU takes"); + let Err(why) = Encoder::open_with(&FakeSource(SharedFake::new(high)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) else { + panic!("Main is not the baseline the SFU takes"); + }; assert!(why.contains("not the baseline"), "{why}"); assert_eq!(choose(&Err(why)), Choice::OpenH264); let mut silent = FakeMft::default(); silent.negotiated_profile = None; - let why = Encoder::open_with(&FakeSource(SharedFake::new(silent)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) - .expect_err("a kept type that names no profile cannot be held to the promise"); + let Err(why) = Encoder::open_with(&FakeSource(SharedFake::new(silent)), Platform::unstarted(), 4, 4, 2_500_000, 30., true) else { + panic!("a kept type that names no profile cannot be held to the promise"); + }; assert!(why.contains("profile"), "{why}"); // Constrained baseline and plain baseline are the promise kept. diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index 1dd4dbd2..1d32032d 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -1081,8 +1081,9 @@ mod tests { #[test] fn a_refused_hardware_encoder_settles_on_the_software_one() { let refused = Err("the hardware encoder encodes H.264 profile 100, not the baseline the SFU takes.".to_owned()); - let why = settle(refused, || Err("OpenH264 could not be downloaded. The call stays a voice call.".to_owned())) - .expect_err("the software answer is the track's"); + let Err(why) = settle(refused, || Err("OpenH264 could not be downloaded. The call stays a voice call.".to_owned())) else { + panic!("the software answer is the track's"); + }; assert!(why.contains("OpenH264"), "{why}"); } From 87e80989a0511da937b8752e23322e498b307a98 Mon Sep 17 00:00:00 2001 From: unRekable Date: Wed, 11 Nov 2026 20:36:42 +0100 Subject: [PATCH 7/7] The answer is asked once, a silent driver is called hung, and the keyframe ask carries a one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The maintainer's re-review of the draft, all four findings: - available() returns what the OnceLock holds before any probe runs, so the "asked once" cache really skips the probe: the frame loops ask it every frame while a slot is empty, and each call now costs a read instead of a thread with CoInitializeEx, MFStartup, MFTEnumEx and MFShutdown in it. - Encoder::encode keeps the moment bytes last came out; about two seconds of silence (STALLED) while frames are still offered is an Err, whose words the Coded failover swaps OpenH264 in on. A driver that stops answering its event queue used to answer every frame a skip forever, and the share froze. - The per-frame logs are said once: the refused bitrate (RealMft::tune), the frames skipped while OpenH264's download lands (Coded::encode), and the machine's "no hardware encoder" answer in needs_openh264, which kept saying the cached refusal on every camera frame. - CODECAPI_AVEncVideoForceKeyFrame goes out as counted(1), where the code had counted(0): Microsoft's docs say a nonzero value codes the next frame a keyframe, and the ask resets itself after that one frame, so zero left the choice to the encoder — no ask at all, and an ask.key could land on a delta. - The merge of origin/main keeps both test sets whole in desktop.rs (main's self-view tests beside the refused-encoder test), and the gate scenario runs the new hung-driver test besides. rustfmt --edition 2021 was run as a parse gate only (no errors in the touched files): stable rustfmt 1.10's defaults restyle the whole sources, including files main itself committed, and CI runs no fmt check — so the files keep the shape they are committed in and the new lines follow it. --- crates/mediad/src/mf_encoder.rs | 77 ++++++++++++++++--- crates/mediad/src/picture/desktop.rs | 20 ++++- .../hardware-encoder-fallback/README.md | 3 +- .../hardware-encoder-fallback/run.sh | 1 + 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/crates/mediad/src/mf_encoder.rs b/crates/mediad/src/mf_encoder.rs index bf526943..16ffda27 100644 --- a/crates/mediad/src/mf_encoder.rs +++ b/crates/mediad/src/mf_encoder.rs @@ -16,7 +16,9 @@ //! an empty [`Encoded`], which the share already treats as a skip; they leave on a later //! frame's pump. The first frame may wait the encoder's warmup for its bits; every later //! one waits a far shorter moment, so a driver that holds frames back never stalls the -//! capture thread. +//! capture thread. A driver that stops sending bits altogether is called hung after +//! [`STALLED`], as an Err the share swaps OpenH264 in on, the way every other refusal of +//! the hardware runs. //! //! The asks of the transform are narrowed to the [`Mft`] trait, and the transform comes //! from a [`Source`]: [`Hardware`] is the machine's own, and the tests hold fakes against @@ -68,6 +70,11 @@ const ANSWER_WAIT: Duration = Duration::from_millis(120); const OUTPUT_WAIT: Duration = Duration::from_millis(10); /// How often the event queue is looked at while waiting for that answer. const POLL: Duration = Duration::from_millis(1); +/// How long the transform may go without a byte out while frames are still offered, +/// before it is called hung: an [`Encoded`] with no bytes is a skip, so a driver that +/// stopped answering its queue would hold the share frozen forever, every frame a skip. +/// The Err is what the share's failover swaps OpenH264 in on. +const STALLED: Duration = Duration::from_secs(2); /// The NV12 stride promised to the encoder, the width rounded up as the hardware wants it. const STRIDE_ALIGN: usize = 16; /// The profiles the SFU is told to expect, as Media Foundation names them: @@ -99,6 +106,11 @@ pub fn choose(probe: &Result<(), String>) -> Choice { /// next open. pub fn available() -> Result<(), String> { static AVAILABLE: OnceLock> = OnceLock::new(); + // An answer held is the answer, with no probe: the frame loops ask this every frame + // while a slot is empty, and each call must be free once the machine has spoken. + if let Some(asked) = AVAILABLE.get() { + return asked.clone(); + } match probe_thread() { Ok(answer) => AVAILABLE.get_or_init(|| answer).clone(), Err(why) => Err(why), @@ -330,7 +342,7 @@ impl Source for Hardware { // `codec` is `None` when the MFT exposes no codec properties, which leaves a // keyframe ask and a bitrate move without their handle; the picture still encodes. let codec: Option = transform.cast().ok(); - Ok(Box::new(RealMft { transform, events, codec })) + Ok(Box::new(RealMft { transform, events, codec, tune_said: false })) } } @@ -340,6 +352,9 @@ struct RealMft { transform: IMFTransform, events: IMFMediaEventGenerator, codec: Option, + /// Whether a refused bitrate has been said, once: the ask comes back every frame + /// while the encoder holds the rate it has, and the log does not follow it there. + tune_said: bool, } /// SAFETY: the MFT is opened, driven, and dropped on one thread (the share's or the @@ -499,9 +514,18 @@ impl Mft for RealMft { // SAFETY: the property is read through the codec handle this encoder holds, and // the variant outlives the call. match unsafe { codec.SetValue(&CODECAPI_AVEncCommonMeanBitRate, &counted(bps)) } { - Ok(()) => true, + Ok(()) => { + self.tune_said = false; + true + } Err(err) => { - tracing::warn!("mediad: the hardware encoder refused a bitrate of {}: {err}", bps); + // The refused ask comes back on every frame that moves the rate, for the + // encoder keeps the rate it has: said once, the next frames stay silent, + // until a rate is taken again and a later refusal is worth its words. + if !self.tune_said { + tracing::warn!("mediad: the hardware encoder refused a bitrate of {}: {err}", bps); + self.tune_said = true; + } false } } @@ -511,10 +535,12 @@ impl Mft for RealMft { let Some(codec) = &self.codec else { return; }; - // The property carries no value; setting it is the ask. An encoder without codec + // One is the ask: the docs say a nonzero value codes the next frame a keyframe + // and the ask resets itself after that one frame, where zero would leave the + // choice to the encoder, which is no ask at all. An encoder without codec // properties keeps its own keyframe pace. - // SAFETY: as in `tune`, with a property whose value is ignored. - if let Err(err) = unsafe { codec.SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &counted(0)) } { + // SAFETY: as in `tune`, with a value the ask is read from. + if let Err(err) = unsafe { codec.SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &counted(1)) } { tracing::debug!("mediad: the hardware encoder refused a keyframe: {err}"); } } @@ -549,6 +575,10 @@ pub struct Encoder { /// Whether a frame has gone in yet: the first may wait the encoder's warmup for its /// bits, every later one waits at most [`OUTPUT_WAIT`]. fed: bool, + /// When the transform last handed bytes over: a driver that stays silent for + /// [`STALLED`] while frames are still offered is hung, and its words fail the encode, + /// which is what the share's failover swaps OpenH264 in on. + answered: Instant, /// COM and Media Foundation, held so they outlive the transform and are let go after /// it (the field after the transform's own). _platform: Platform, @@ -586,7 +616,7 @@ impl Encoder { mft.set_input_type(width, height, stride, frame, fps)?; mft.configure(bps); mft.begin()?; - let mut encoder = Encoder { mft, width, height, stride, frame, fps, bps, wants_input: false, fed: false, _platform: platform }; + let mut encoder = Encoder { mft, width, height, stride, frame, fps, bps, wants_input: false, fed: false, answered: Instant::now(), _platform: platform }; // The first input slot is usually open before the first frame; wait a moment // for it, so the first frame of a share is fed at once and not dropped. let mut nothing = Vec::new(); @@ -604,7 +634,8 @@ impl Encoder { /// empty, like a skip; an encoded frame comes back marked when it is a keyframe, so /// the share sends it as one. After the first frame is in, the wait for a frame's own /// bits is bounded well under a frame's time, so the capture thread is never held - /// long. + /// long. A driver that sends no bits at all for [`STALLED`] while frames keep being + /// offered is called hung, as an Err: the share's failover takes OpenH264 on it. pub fn encode(&mut self, planes: &Planes, ask: &Ask) -> Result { if planes.width != self.width || planes.height != self.height { return Err(format!("The encoder was opened for {}×{} and handed {}×{}.", self.width, self.height, planes.width, planes.height)); @@ -641,6 +672,16 @@ impl Encoder { } if bytes.is_empty() { key = false; + // The frames are still being offered, and for [`STALLED`] not one of their + // bits has come out: a driver that stopped answering its queue would say + // every frame a skip forever, and the share would freeze looking skipped. + // Said as an Err, this is the swap to OpenH264's cue, the way every other + // failure of the hardware runs. + if self.answered.elapsed() >= STALLED { + return Err("the hardware encoder has sent no bits for two seconds, with frames going in.".to_owned()); + } + } else { + self.answered = Instant::now(); } Ok(Encoded { bytes, key }) } @@ -1191,4 +1232,22 @@ mod tests { assert!(!why.is_empty(), "a refusal says why"); } } + + /// A driver that stops sending bits is hung, not patient: before [`STALLED`] the + /// frames come back as the skips they are, and the silence past it, with frames still + /// being offered, is an Err — the words the share's failover swaps OpenH264 in on. + /// The fake holds the same silence a real driver hangs with: slots keep opening and + /// the frames go in, but nothing ever comes out. + #[test] + fn a_driver_that_goes_silent_is_called_hung_after_two_seconds_of_it() { + let mut encoder = opened(SharedFake::new(baseline_fake())); + let skip = encoder.encode(&planes(), &Ask::default()).expect("fresh silence is a skip, not a failure"); + assert!(skip.bytes.is_empty(), "the empty frame before the stall goes out as a skip"); + encoder.answered = Instant::now() - STALLED; + let Err(why) = encoder.encode(&planes(), &Ask::default()) else { + panic!("two seconds of silence with frames going in is a hung driver"); + }; + assert!(why.contains("no bits"), "{why}"); + assert_eq!(choose(&Err(why)), Choice::OpenH264, "the words are the swap's cue"); + } } diff --git a/crates/mediad/src/picture/desktop.rs b/crates/mediad/src/picture/desktop.rs index ddf0eb3f..ea0256e9 100644 --- a/crates/mediad/src/picture/desktop.rs +++ b/crates/mediad/src/picture/desktop.rs @@ -60,6 +60,10 @@ struct Coded { /// second does not say it a second a time. #[cfg(windows)] said: bool, + /// Whether the wait for OpenH264's takeover has been said once, so the frames skipped + /// while the download lands do not say it one a frame. + #[cfg(windows)] + said_skip: bool, } /// One of the two encoders behind [`Coded`]: the machine's own hardware, or Cisco's @@ -154,7 +158,7 @@ impl Coded { #[cfg(windows)] fn with(backend: Backend, width: u32, height: u32, bps: u32, fps: f32, screen: bool) -> Coded { - Coded { backend, width, height, bps, fps, screen, said: false } + Coded { backend, width, height, bps, fps, screen, said: false, said_skip: false } } #[cfg(not(windows))] @@ -203,7 +207,12 @@ impl Coded { return self.backend.encode(planes, &forced); } Err(why) => { - tracing::debug!("mediad: OpenH264 is not ready to take over ({why}); this frame is skipped."); + // The download lands once; the frames it skips are not said + // one a frame while it does. + if !self.said_skip { + tracing::debug!("mediad: OpenH264 is not ready to take over ({why}); this frame is skipped."); + self.said_skip = true; + } return Ok(Encoded::default()); } } @@ -238,7 +247,12 @@ fn needs_openh264() -> bool { match mf_encoder::available() { Ok(()) => false, Err(why) => { - tracing::debug!("mediad: no Media Foundation encoder ({why}); OpenH264 stays the codec."); + // The answer is the machine's and does not change; the frames keep asking + // while a slot is empty, and the words are said on the first ask only. + static SAID: OnceLock<()> = OnceLock::new(); + if SAID.set(()).is_ok() { + tracing::debug!("mediad: no Media Foundation encoder ({why}); OpenH264 stays the codec."); + } true } } diff --git a/e2e/scenarios/hardware-encoder-fallback/README.md b/e2e/scenarios/hardware-encoder-fallback/README.md index 1669d140..21f73aa3 100644 --- a/e2e/scenarios/hardware-encoder-fallback/README.md +++ b/e2e/scenarios/hardware-encoder-fallback/README.md @@ -5,7 +5,8 @@ hardware and no network: the scenario drives the tests of `crates/mediad/src/mf_encoder.rs` and the fallback seam in `picture/desktop.rs` — the encoder selection, the fake-MFT seam that holds the open (the H.264 output type refused, a driver that kept Main/High or named no profile, a keyframe that only counts when the -sample says `MFSampleExtension_CleanPoint`, the frame loop with its asks), the pure NV12 +sample says `MFSampleExtension_CleanPoint`, the frame loop with its asks, a driver that +goes silent called hung after two seconds so the failover runs), the pure NV12 and Annex B helpers, and the place a refusal of the hardware becomes the OpenH264 fallback. diff --git a/e2e/scenarios/hardware-encoder-fallback/run.sh b/e2e/scenarios/hardware-encoder-fallback/run.sh index 583bb332..55436bf9 100644 --- a/e2e/scenarios/hardware-encoder-fallback/run.sh +++ b/e2e/scenarios/hardware-encoder-fallback/run.sh @@ -28,6 +28,7 @@ run mf_encoder::tests::an_encoder_that_refuses_the_output_type_is_failed_in_its_ run mf_encoder::tests::a_profile_other_than_the_baseline_promised_is_refused run mf_encoder::tests::a_frame_is_a_keyframe_only_when_the_sample_says_it_is_a_clean_point run mf_encoder::tests::an_open_encoder_takes_frames_and_holds_the_asks +run mf_encoder::tests::a_driver_that_goes_silent_is_called_hung_after_two_seconds_of_it run mf_encoder::tests::a_closed_encoder_ends_its_stream run mf_encoder::tests::the_probe_answers_on_a_thread_of_its_own run mf_encoder::tests::nv12_is_the_y_plane_and_the_two_chroma_planes_interleaved