From f79692347a3f8a173a8ad92d9f6e2b7088f50442 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:46 -0700 Subject: [PATCH 01/38] Add additional deps --- Cargo.lock | 12 ++++++++++++ livekit-capture/Cargo.toml | 3 +++ 2 files changed, 15 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 73b434a8a..bf24c3b9e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3828,12 +3828,14 @@ dependencies = [ name = "livekit-capture" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "chrono", "gstreamer", "gstreamer-app", "livekit", "log", + "md-5", "pollster", "schemars", "serde", @@ -4201,6 +4203,16 @@ dependencies = [ "rayon", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if 1.0.4", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 711665cb7..8f015585d 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -8,10 +8,12 @@ edition.workspace = true repository.workspace = true [dependencies] +base64 = { version = "0.22", optional = true } bytes = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } +md-5 = { version = "0.10", optional = true } livekit = { workspace = true } log = { workspace = true } pollster = { version = "0.4", optional = true } @@ -36,3 +38,4 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] +source-rtsp = ["dep:base64", "dep:md-5"] From 7d188b29c0feced93d6495392690c0464bcab492 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:48 -0700 Subject: [PATCH 02/38] Implement bitstream readers --- livekit-capture/src/sources/rtsp/bits.rs | 164 +++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/bits.rs diff --git a/livekit-capture/src/sources/rtsp/bits.rs b/livekit-capture/src/sources/rtsp/bits.rs new file mode 100644 index 000000000..2cfdd7b2f --- /dev/null +++ b/livekit-capture/src/sources/rtsp/bits.rs @@ -0,0 +1,164 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bitstream readers shared by the RTP depacketizers and the codec header +//! parsers. + +/// MSB-first bit reader over a byte slice. Every read returns `None` past +/// the end of the input. +#[derive(Debug, Clone)] +pub(super) struct BitReader<'a> { + bytes: &'a [u8], + bit_offset: usize, +} + +impl<'a> BitReader<'a> { + /// Creates a reader positioned at the first bit. + pub(super) fn new(bytes: &'a [u8]) -> Self { + Self { bytes, bit_offset: 0 } + } + + /// Reads one bit. + pub(super) fn read_bit(&mut self) -> Option { + let byte = *self.bytes.get(self.bit_offset / 8)?; + let bit = (byte >> (7 - self.bit_offset % 8)) & 0x01; + self.bit_offset += 1; + Some(bit) + } + + /// Reads one bit as a flag. + pub(super) fn read_flag(&mut self) -> Option { + self.read_bit().map(|bit| bit != 0) + } + + /// Reads up to 32 bits MSB-first. + pub(super) fn read_bits(&mut self, bits: u32) -> Option { + debug_assert!(bits <= 32); + let mut value = 0u32; + for _ in 0..bits { + value = (value << 1) | u32::from(self.read_bit()?); + } + Some(value) + } + + /// Skips over `bits` bits. + pub(super) fn skip_bits(&mut self, bits: usize) -> Option<()> { + let next = self.bit_offset.checked_add(bits)?; + if next > self.bytes.len() * 8 { + return None; + } + self.bit_offset = next; + Some(()) + } + + /// Reads an unsigned Exp-Golomb code (`ue(v)` in H.264/H.265). + pub(super) fn read_ue(&mut self) -> Option { + let mut leading_zeros = 0u32; + while self.read_bit()? == 0 { + leading_zeros += 1; + if leading_zeros > 31 { + return None; + } + } + let suffix = self.read_bits(leading_zeros)?; + (1u32 << leading_zeros).checked_sub(1)?.checked_add(suffix) + } + + /// Reads a signed Exp-Golomb code (`se(v)` in H.264/H.265) and discards + /// the value. + pub(super) fn skip_se(&mut self) -> Option<()> { + self.read_ue().map(|_| ()) + } +} + +/// Reads an AV1/LEB128 length from `bytes` at `cursor`, advancing it. +pub(super) fn read_leb128(bytes: &[u8], cursor: &mut usize) -> Option { + let mut value = 0usize; + let mut shift = 0usize; + loop { + let &byte = bytes.get(*cursor)?; + *cursor += 1; + value |= usize::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Some(value); + } + shift += 7; + if shift >= usize::BITS as usize { + return None; + } + } +} + +/// Appends `value` to `out` as LEB128. +pub(super) fn write_leb128(mut value: usize, out: &mut Vec) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + break; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_bits_msb_first() { + let mut reader = BitReader::new(&[0b1011_0001, 0b1000_0000]); + assert_eq!(reader.read_bit(), Some(1)); + assert_eq!(reader.read_bits(3), Some(0b011)); + assert_eq!(reader.read_bits(5), Some(0b0001_1)); + assert!(reader.skip_bits(7).is_some()); + assert_eq!(reader.read_bit(), None); + } + + #[test] + fn reads_exp_golomb_codes() { + // ue(v) codes: 1 -> 0, 010 -> 1, 011 -> 2, 00100 -> 3. + let mut reader = BitReader::new(&[0b1010_0110, 0b0100_0000]); + assert_eq!(reader.read_ue(), Some(0)); + assert_eq!(reader.read_ue(), Some(1)); + assert_eq!(reader.read_ue(), Some(2)); + assert_eq!(reader.read_ue(), Some(3)); + } + + #[test] + fn exp_golomb_past_end_is_none() { + let mut reader = BitReader::new(&[0b0000_0000]); + assert_eq!(reader.read_ue(), None); + } + + #[test] + fn leb128_round_trips() { + for value in [0usize, 1, 127, 128, 300, 16_383, 16_384, usize::from(u16::MAX)] { + let mut encoded = Vec::new(); + write_leb128(value, &mut encoded); + let mut cursor = 0; + assert_eq!(read_leb128(&encoded, &mut cursor), Some(value)); + assert_eq!(cursor, encoded.len()); + } + } + + #[test] + fn leb128_rejects_truncated_input() { + let mut cursor = 0; + assert_eq!(read_leb128(&[0x80], &mut cursor), None); + } +} From af41d5eed710ca6ec901419f12800ff0aa111874 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:49 -0700 Subject: [PATCH 03/38] Implement RTP access unit assembly --- livekit-capture/src/sources/rtsp/rtp/mod.rs | 657 ++++++++++++++++++++ 1 file changed, 657 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/rtp/mod.rs diff --git a/livekit-capture/src/sources/rtsp/rtp/mod.rs b/livekit-capture/src/sources/rtsp/rtp/mod.rs new file mode 100644 index 000000000..b72c7228a --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/mod.rs @@ -0,0 +1,657 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTP depacketization into encoded access units. +//! +//! [`RtpAccessUnitAssembler`] reassembles the RTP payload formats of the +//! supported codecs and recovers from packet loss by discarding the +//! interrupted access unit and gating output on the next keyframe. + +mod av1; +mod h26x; +mod vpx; + +use std::collections::VecDeque; + +use thiserror::Error; + +use crate::{ + encoded::{ + h26x::H26xParseError, EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + }, + primitive::VideoResolution, +}; + +/// Out-of-band H.26x parameter sets, decoded from SDP `fmtp` attributes. +/// +/// The assembler prepends missing parameter sets to keyframe access units so +/// every published keyframe is self-contained; see +/// [`RtpAccessUnitAssembler::finish_current`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(super) struct H26xParameterSets { + /// H.265 video parameter sets (NAL type 32). + pub(super) vps: Vec>, + /// Sequence parameter sets (H.264 NAL type 7, H.265 NAL type 33). + pub(super) sps: Vec>, + /// Picture parameter sets (H.264 NAL type 8, H.265 NAL type 34). + pub(super) pps: Vec>, +} + +impl H26xParameterSets { + /// Returns `true` when no parameter sets were provided. + #[cfg(test)] + pub(super) fn is_empty(&self) -> bool { + self.vps.is_empty() && self.sps.is_empty() && self.pps.is_empty() + } +} + +/// Parsed RTP packet header and payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct RtpPacket<'a> { + /// RTP marker bit. + pub(super) marker: bool, + /// RTP payload type. + pub(super) payload_type: u8, + /// RTP sequence number. + pub(super) sequence_number: u16, + /// RTP timestamp. + pub(super) timestamp: u32, + /// RTP SSRC. + pub(super) ssrc: u32, + /// RTP payload bytes. + pub(super) payload: &'a [u8], +} + +impl<'a> RtpPacket<'a> { + /// Parses a single RTP packet. + pub(super) fn parse(bytes: &'a [u8]) -> Result { + if bytes.len() < 12 { + return Err(RtpDepacketizerError::PacketTooShort); + } + if bytes[0] >> 6 != 2 { + return Err(RtpDepacketizerError::UnsupportedVersion(bytes[0] >> 6)); + } + + let has_padding = (bytes[0] & 0x20) != 0; + let has_extension = (bytes[0] & 0x10) != 0; + let csrc_count = (bytes[0] & 0x0f) as usize; + let mut payload_start = 12 + csrc_count * 4; + if bytes.len() < payload_start { + return Err(RtpDepacketizerError::PacketTooShort); + } + + if has_extension { + if bytes.len() < payload_start + 4 { + return Err(RtpDepacketizerError::PacketTooShort); + } + let extension_words = + u16::from_be_bytes([bytes[payload_start + 2], bytes[payload_start + 3]]) as usize; + payload_start += 4 + extension_words * 4; + if bytes.len() < payload_start { + return Err(RtpDepacketizerError::PacketTooShort); + } + } + + let payload_end = if has_padding { + let Some(padding) = bytes.last().copied() else { + return Err(RtpDepacketizerError::PacketTooShort); + }; + let padding = padding as usize; + if padding == 0 || bytes.len() < payload_start + padding { + return Err(RtpDepacketizerError::PacketTooShort); + } + bytes.len() - padding + } else { + bytes.len() + }; + + Ok(Self { + marker: (bytes[1] & 0x80) != 0, + payload_type: bytes[1] & 0x7f, + sequence_number: u16::from_be_bytes([bytes[2], bytes[3]]), + timestamp: u32::from_be_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), + ssrc: u32::from_be_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]), + payload: &bytes[payload_start..payload_end], + }) + } +} + +/// Maps RTP timestamps to capture timestamps in microseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RtpTimestampMapper { + clock_rate: u32, + last_rtp_timestamp: Option, + extended_ticks: i64, + base_timestamp_us: i64, +} + +impl RtpTimestampMapper { + /// Creates an RTP timestamp mapper with a non-zero clock rate. + fn new(clock_rate: u32, base_timestamp_us: i64) -> Result { + if clock_rate == 0 { + return Err(RtpDepacketizerError::InvalidClockRate); + } + Ok(Self { clock_rate, last_rtp_timestamp: None, extended_ticks: 0, base_timestamp_us }) + } + + /// Maps an RTP timestamp to microseconds, unwrapping `u32` RTP timestamp + /// rollover so mapped timestamps stay monotonic across any number of wraps. + fn map(&mut self, rtp_timestamp: u32) -> i64 { + let last = *self.last_rtp_timestamp.get_or_insert(rtp_timestamp); + self.last_rtp_timestamp = Some(rtp_timestamp); + // Reinterpreting the wrapped u32 delta as i32 picks the nearest extended + // timestamp, which unwraps rollover while tolerating small backwards + // jumps from reordered packets. + let delta_ticks = i64::from(rtp_timestamp.wrapping_sub(last) as i32); + self.extended_ticks = self.extended_ticks.saturating_add(delta_ticks); + + let extended_us = i128::from(self.extended_ticks) * 1_000_000 / i128::from(self.clock_rate); + let extended_us = extended_us.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64; + self.base_timestamp_us.saturating_add(extended_us) + } +} + +/// Error returned by RTP depacketization and access-unit assembly. +#[derive(Debug, Error, PartialEq, Eq)] +pub(super) enum RtpDepacketizerError { + /// RTP packet is shorter than its declared header. + #[error("RTP packet is too short")] + PacketTooShort, + /// RTP version is not supported. + #[error("unsupported RTP version {0}")] + UnsupportedVersion(u8), + /// RTP clock rate must be non-zero. + #[error("RTP clock rate must be non-zero")] + InvalidClockRate, + /// RTP payload format is unsupported or malformed. + #[error("unsupported or malformed RTP payload")] + UnsupportedPayload, + /// RTP fragmentation state was invalid. + #[error("invalid RTP fragmentation sequence")] + InvalidFragment, + /// The payload descriptor is unsupported by the single-layer depacketizer. + #[error("unsupported RTP payload descriptor")] + UnsupportedPayloadDescriptor, + /// Assembled NAL units could not form an access unit. + #[error(transparent)] + H26x(#[from] H26xParseError), +} + +/// Reassembles the RTP packets of one video stream into encoded access units. +/// +/// Packets whose payload type or SSRC does not match the negotiated stream +/// are ignored. Packet loss is recovered internally: gaps and truncated +/// fragments drop the interrupted access unit and gate output on the next +/// keyframe instead of returning an error. +#[derive(Debug, Clone)] +pub(super) struct RtpAccessUnitAssembler { + codec: EncodedVideoCodec, + payload_type: u8, + /// SSRC latched from the first matching packet; later packets from other + /// sources on the same channel are dropped. + ssrc: Option, + resolution: VideoResolution, + parameter_sets: H26xParameterSets, + timestamp_mapper: RtpTimestampMapper, + expected_sequence_number: Option, + current: Option, + fragment: Option, + current_frame: Option, + av1_fragment: Option, + ready: VecDeque, + awaiting_keyframe: bool, + logged_ssrc_mismatch: bool, + warned_missing_parameter_sets: bool, + sequence_gaps: u64, + dropped_access_units: u64, +} + +#[derive(Debug, Clone)] +struct PartialAccessUnit { + rtp_timestamp: u32, + timestamp_us: i64, + nal_units: Vec>, +} + +#[derive(Debug, Clone)] +struct FragmentState { + rtp_timestamp: u32, + nal_unit: Vec, +} + +#[derive(Debug, Clone)] +struct PartialFrame { + rtp_timestamp: u32, + timestamp_us: i64, + payload: Vec, + frame_type: Option, + av1_reduced_still_picture_header: Option, +} + +#[derive(Debug, Clone)] +struct Av1FragmentState { + rtp_timestamp: u32, + obu: Vec, +} + +/// Packet-loss recovery counters for an [`RtpAccessUnitAssembler`]. +#[cfg(test)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) struct RtpDepacketizerStats { + pub(super) sequence_gaps: u64, + pub(super) dropped_access_units: u64, + pub(super) awaiting_keyframe: bool, +} + +impl RtpAccessUnitAssembler { + /// Creates an RTP access-unit assembler for one negotiated video stream. + pub(super) fn new( + codec: EncodedVideoCodec, + payload_type: u8, + clock_rate: u32, + parameter_sets: H26xParameterSets, + resolution: VideoResolution, + ) -> Result { + Ok(Self { + codec, + payload_type, + ssrc: None, + resolution, + parameter_sets, + timestamp_mapper: RtpTimestampMapper::new(clock_rate, 0)?, + expected_sequence_number: None, + current: None, + fragment: None, + current_frame: None, + av1_fragment: None, + ready: VecDeque::new(), + awaiting_keyframe: false, + logged_ssrc_mismatch: false, + warned_missing_parameter_sets: false, + sequence_gaps: 0, + dropped_access_units: 0, + }) + } + + /// Sets the resolution stamped on access units assembled from now on. + pub(super) fn set_resolution(&mut self, resolution: VideoResolution) { + self.resolution = resolution; + } + + /// Returns the next completed access unit, if any. + pub(super) fn pop_ready(&mut self) -> Option { + self.ready.pop_front() + } + + /// Returns packet-loss recovery counters. + #[cfg(test)] + pub(super) fn stats(&self) -> RtpDepacketizerStats { + RtpDepacketizerStats { + sequence_gaps: self.sequence_gaps, + dropped_access_units: self.dropped_access_units, + awaiting_keyframe: self.awaiting_keyframe, + } + } + + /// Pushes one encoded RTP packet; completed access units become available + /// through [`Self::pop_ready`]. + pub(super) fn push(&mut self, bytes: &[u8]) -> Result<(), RtpDepacketizerError> { + let packet = RtpPacket::parse(bytes)?; + if packet.payload_type != self.payload_type { + return Ok(()); + } + match self.ssrc { + None => self.ssrc = Some(packet.ssrc), + Some(ssrc) if ssrc != packet.ssrc => { + if !self.logged_ssrc_mismatch { + log::debug!( + "dropping RTP packets from unexpected SSRC {:#010x}; \ + the stream is locked to SSRC {ssrc:#010x}", + packet.ssrc, + ); + self.logged_ssrc_mismatch = true; + } + return Ok(()); + } + Some(_) => {} + } + + self.check_sequence(packet.sequence_number); + self.finish_on_timestamp_change(packet.timestamp)?; + + match self.codec { + EncodedVideoCodec::H264 => self.push_h264_payload(&packet)?, + EncodedVideoCodec::H265 => self.push_h265_payload(&packet)?, + EncodedVideoCodec::VP8 => self.push_vp8_payload(&packet)?, + EncodedVideoCodec::VP9 => self.push_vp9_payload(&packet)?, + EncodedVideoCodec::AV1 => self.push_av1_payload(&packet)?, + } + + if packet.marker { + if self.fragment.is_some() || self.av1_fragment.is_some() { + // The marker closed the access unit before the open fragment's + // end arrived, so its tail packets were lost. + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + self.finish_pending()?; + } + Ok(()) + } + + fn check_sequence(&mut self, sequence_number: u16) { + let Some(expected) = self.expected_sequence_number.replace(sequence_number.wrapping_add(1)) + else { + return; + }; + if sequence_number == expected { + return; + } + + self.sequence_gaps += 1; + self.discard_in_progress(); + } + + /// Completes the pending access unit when a packet starts a new RTP + /// timestamp without a marker having closed the previous one. + /// + /// Some producers never set the RTP marker bit; on a contiguous sequence, + /// a timestamp change still proves the previous access unit is complete. + /// A lost marker packet is a sequence gap instead, which discards the + /// interrupted access unit before this check runs. + fn finish_on_timestamp_change( + &mut self, + rtp_timestamp: u32, + ) -> Result<(), RtpDepacketizerError> { + let pending = self + .current + .as_ref() + .map(|current| current.rtp_timestamp) + .or_else(|| self.current_frame.as_ref().map(|frame| frame.rtp_timestamp)) + .or_else(|| self.fragment.as_ref().map(|fragment| fragment.rtp_timestamp)) + .or_else(|| self.av1_fragment.as_ref().map(|fragment| fragment.rtp_timestamp)); + let Some(pending) = pending else { + return Ok(()); + }; + if pending == rtp_timestamp { + return Ok(()); + } + if self.fragment.is_some() || self.av1_fragment.is_some() { + // A new frame began while a fragment was open: its end was lost. + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + self.finish_pending() + } + + /// Completes the pending access unit or frame for the source's codec. + fn finish_pending(&mut self) -> Result<(), RtpDepacketizerError> { + match self.codec { + EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => self.finish_current(), + _ => self.finish_current_frame(), + } + } + + /// Discards all partially assembled state and gates output on the next keyframe. + fn discard_in_progress(&mut self) { + self.current = None; + self.fragment = None; + self.current_frame = None; + self.av1_fragment = None; + self.awaiting_keyframe = true; + } + + /// Queues a completed access unit, dropping it while loss recovery gates + /// output on the next keyframe. + fn enqueue(&mut self, access_unit: OwnedEncodedAccessUnit) { + if self.awaiting_keyframe { + if access_unit.frame_type != EncodedFrameType::Key { + self.dropped_access_units += 1; + return; + } + self.awaiting_keyframe = false; + } + self.ready.push_back(access_unit); + } + + fn current_mut( + &mut self, + rtp_timestamp: u32, + ) -> Result<&mut PartialAccessUnit, RtpDepacketizerError> { + if self.current.as_ref().is_some_and(|current| current.rtp_timestamp != rtp_timestamp) { + // Unreachable after `finish_on_timestamp_change`; kept as a + // defensive reset. + self.current = None; + self.fragment = None; + } + + if self.current.is_none() { + let timestamp_us = self.timestamp_mapper.map(rtp_timestamp); + self.current = + Some(PartialAccessUnit { rtp_timestamp, timestamp_us, nal_units: Vec::new() }); + } + + self.current.as_mut().ok_or(RtpDepacketizerError::InvalidFragment) + } + + fn current_frame_mut( + &mut self, + rtp_timestamp: u32, + ) -> Result<&mut PartialFrame, RtpDepacketizerError> { + if self.current_frame.as_ref().is_some_and(|current| current.rtp_timestamp != rtp_timestamp) + { + // Unreachable after `finish_on_timestamp_change`; kept as a + // defensive reset. + self.current_frame = None; + self.av1_fragment = None; + } + + if self.current_frame.is_none() { + let timestamp_us = self.timestamp_mapper.map(rtp_timestamp); + self.current_frame = Some(PartialFrame { + rtp_timestamp, + timestamp_us, + payload: Vec::new(), + frame_type: None, + av1_reduced_still_picture_header: None, + }); + } + + self.current_frame.as_mut().ok_or(RtpDepacketizerError::InvalidFragment) + } + + /// Completes the pending VP8/VP9/AV1 frame and queues it. + fn finish_current_frame(&mut self) -> Result<(), RtpDepacketizerError> { + let Some(current) = self.current_frame.take() else { + return Ok(()); + }; + if current.payload.is_empty() { + return Ok(()); + } + + let access_unit = OwnedEncodedAccessUnit::new( + self.codec, + current.payload, + current.timestamp_us, + current.frame_type.unwrap_or(EncodedFrameType::Delta), + self.resolution, + ); + self.enqueue(access_unit); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(super) fn rtp_packet( + sequence_number: u16, + timestamp: u32, + marker: bool, + payload: &[u8], + ) -> Vec { + rtp_packet_from(sequence_number, timestamp, marker, 96, 0x1122_3344, payload) + } + + pub(super) fn rtp_packet_from( + sequence_number: u16, + timestamp: u32, + marker: bool, + payload_type: u8, + ssrc: u32, + payload: &[u8], + ) -> Vec { + let mut packet = Vec::with_capacity(12 + payload.len()); + packet.push(0x80); + packet.push(if marker { 0x80 | payload_type } else { payload_type }); + packet.extend_from_slice(&sequence_number.to_be_bytes()); + packet.extend_from_slice(×tamp.to_be_bytes()); + packet.extend_from_slice(&ssrc.to_be_bytes()); + packet.extend_from_slice(payload); + packet + } + + pub(super) fn assembler(codec: EncodedVideoCodec) -> RtpAccessUnitAssembler { + RtpAccessUnitAssembler::new( + codec, + 96, + 90_000, + H26xParameterSets::default(), + VideoResolution::new(640, 480), + ) + .unwrap() + } + + pub(super) fn push_one( + assembler: &mut RtpAccessUnitAssembler, + bytes: &[u8], + ) -> Option { + assembler.push(bytes).unwrap(); + assembler.pop_ready() + } + + #[test] + fn parses_rtp_packet_header() { + let bytes = rtp_packet(7, 90_000, true, &[0x65, 1, 2]); + let packet = RtpPacket::parse(&bytes).unwrap(); + assert!(packet.marker); + assert_eq!(packet.payload_type, 96); + assert_eq!(packet.sequence_number, 7); + assert_eq!(packet.timestamp, 90_000); + assert_eq!(packet.payload, &[0x65, 1, 2]); + } + + #[test] + fn maps_rtp_timestamp_rollover() { + let mut mapper = RtpTimestampMapper::new(90_000, 1_000).unwrap(); + assert_eq!(mapper.map(u32::MAX - 89), 1_000); + assert_eq!(mapper.map(0), 2_000); + } + + #[test] + fn maps_rtp_timestamps_across_multiple_rollovers() { + let mut mapper = RtpTimestampMapper::new(90_000, 0).unwrap(); + let step = 1u32 << 30; + let mut rtp_timestamp = 0u32; + let mut last_us = mapper.map(rtp_timestamp); + for _ in 0..20 { + rtp_timestamp = rtp_timestamp.wrapping_add(step); + let mapped_us = mapper.map(rtp_timestamp); + assert!(mapped_us > last_us, "mapped timestamps must stay monotonic"); + last_us = mapped_us; + } + assert_eq!(last_us, (20i64 << 30) * 1_000_000 / 90_000); + } + + #[test] + fn maps_reordered_rtp_timestamps() { + let mut mapper = RtpTimestampMapper::new(90_000, 1_000).unwrap(); + assert_eq!(mapper.map(9_000), 1_000); + assert_eq!(mapper.map(18_000), 101_000); + // A late packet maps behind the stream without disturbing what follows. + assert_eq!(mapper.map(15_000), 67_666); + assert_eq!(mapper.map(27_000), 201_000); + } + + #[test] + fn rejects_zero_clock_rate() { + assert_eq!( + RtpTimestampMapper::new(0, 0).unwrap_err(), + RtpDepacketizerError::InvalidClockRate + ); + } + + #[test] + fn ignores_other_payload_types() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // An interloping payload type must not count as a sequence gap. + let other = rtp_packet_from(50, 12_000, true, 97, 0x1122_3344, &[0x65, 9]); + let first = rtp_packet(10, 12_000, false, &[0x65, 1, 2]); + let second = rtp_packet(11, 12_000, true, &[0x41, 3]); + + assert!(push_one(&mut assembler, &first).is_none()); + assert!(push_one(&mut assembler, &other).is_none()); + let access_unit = push_one(&mut assembler, &second).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2, 0, 0, 0, 1, 0x41, 3]); + assert_eq!(assembler.stats().sequence_gaps, 0); + } + + #[test] + fn locks_to_first_ssrc() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let first = rtp_packet_from(10, 12_000, true, 96, 0xaaaa_aaaa, &[0x65, 1]); + let intruder = rtp_packet_from(70, 15_000, true, 96, 0xbbbb_bbbb, &[0x65, 2]); + let second = rtp_packet_from(11, 15_000, true, 96, 0xaaaa_aaaa, &[0x41, 3]); + + assert!(push_one(&mut assembler, &first).unwrap().frame_type == EncodedFrameType::Key); + assert!(push_one(&mut assembler, &intruder).is_none()); + let access_unit = push_one(&mut assembler, &second).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(assembler.stats().sequence_gaps, 0); + } + + #[test] + fn timestamp_change_completes_marker_less_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // The producer never sets the marker bit. + let first = rtp_packet(10, 12_000, false, &[0x65, 1, 2]); + let second = rtp_packet(11, 15_000, false, &[0x41, 3, 4]); + let third = rtp_packet(12, 18_000, false, &[0x41, 5, 6]); + + assert!(push_one(&mut assembler, &first).is_none()); + let key = push_one(&mut assembler, &second).unwrap(); + assert_eq!(key.frame_type, EncodedFrameType::Key); + assert_eq!(key.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + + let delta = push_one(&mut assembler, &third).unwrap(); + assert_eq!(delta.frame_type, EncodedFrameType::Delta); + assert_eq!(delta.payload.as_ref(), &[0, 0, 0, 1, 0x41, 3, 4]); + } + + #[test] + fn timestamp_change_with_open_fragment_drops_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + // The fragment end never arrives; the next frame starts instead. + let next = rtp_packet(11, 15_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &next).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(assembler.stats().dropped_access_units, 1); + } +} From 59995f326cc9c0b7920573c8d68b401c42f4b047 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:49 -0700 Subject: [PATCH 04/38] Implement H26x depacketization --- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 475 +++++++++++++++++++ 1 file changed, 475 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/rtp/h26x.rs diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs new file mode 100644 index 000000000..f9e668810 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -0,0 +1,475 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! H.264 (RFC 6184) and H.265 (RFC 7798) RTP payload handling. + +use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::encoded::{h26x::access_unit_from_nalus, EncodedFrameType, EncodedVideoCodec}; + +impl RtpAccessUnitAssembler { + pub(super) fn push_h264_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let payload = packet.payload; + let Some(&header) = payload.first() else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + let nal_type = header & 0x1f; + + match nal_type { + 1..=23 => self.current_mut(packet.timestamp)?.nal_units.push(payload.to_vec()), + 24 => self.push_h26x_aggregation(packet.timestamp, &payload[1..])?, + 28 => self.push_h264_fu_a(packet.timestamp, payload)?, + _ => return Err(RtpDepacketizerError::UnsupportedPayload), + } + + Ok(()) + } + + pub(super) fn push_h265_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let payload = packet.payload; + if payload.len() < 2 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + let nal_type = (payload[0] >> 1) & 0x3f; + + match nal_type { + 0..=47 => self.current_mut(packet.timestamp)?.nal_units.push(payload.to_vec()), + 48 => self.push_h26x_aggregation(packet.timestamp, &payload[2..])?, + 49 => self.push_h265_fragment(packet.timestamp, payload)?, + _ => return Err(RtpDepacketizerError::UnsupportedPayload), + } + + Ok(()) + } + + /// Unpacks the length-prefixed NAL units of an H.264 STAP-A or H.265 AP + /// payload, whose aggregation headers the caller has already stripped. + fn push_h26x_aggregation( + &mut self, + rtp_timestamp: u32, + payload: &[u8], + ) -> Result<(), RtpDepacketizerError> { + let mut cursor = 0; + while cursor < payload.len() { + if payload.len() < cursor + 2 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + let len = u16::from_be_bytes([payload[cursor], payload[cursor + 1]]) as usize; + cursor += 2; + if len == 0 || payload.len() < cursor + len { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + self.current_mut(rtp_timestamp)?.nal_units.push(payload[cursor..cursor + len].to_vec()); + cursor += len; + } + Ok(()) + } + + fn push_h264_fu_a( + &mut self, + rtp_timestamp: u32, + payload: &[u8], + ) -> Result<(), RtpDepacketizerError> { + if payload.len() < 2 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let indicator = payload[0]; + let header = payload[1]; + let start = (header & 0x80) != 0; + let end = (header & 0x40) != 0; + let nal_type = header & 0x1f; + if nal_type == 0 || nal_type > 23 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + if start { + let mut nal_unit = Vec::with_capacity(1 + payload.len().saturating_sub(2)); + nal_unit.push((indicator & 0xe0) | nal_type); + nal_unit.extend_from_slice(&payload[2..]); + self.fragment = Some(FragmentState { rtp_timestamp, nal_unit }); + return Ok(()); + } + + let Some(fragment) = + self.fragment.as_mut().filter(|fragment| fragment.rtp_timestamp == rtp_timestamp) + else { + // A continuation without its start means the preceding packets were lost. + self.discard_in_progress(); + return Ok(()); + }; + fragment.nal_unit.extend_from_slice(&payload[2..]); + + if end { + let nal_unit = + self.fragment.take().ok_or(RtpDepacketizerError::InvalidFragment)?.nal_unit; + self.current_mut(rtp_timestamp)?.nal_units.push(nal_unit); + } + Ok(()) + } + + fn push_h265_fragment( + &mut self, + rtp_timestamp: u32, + payload: &[u8], + ) -> Result<(), RtpDepacketizerError> { + if payload.len() < 3 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let fu_header = payload[2]; + let start = (fu_header & 0x80) != 0; + let end = (fu_header & 0x40) != 0; + let nal_type = fu_header & 0x3f; + if nal_type > 47 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + if start { + let mut nal_unit = Vec::with_capacity(2 + payload.len().saturating_sub(3)); + nal_unit.push((payload[0] & 0x81) | (nal_type << 1)); + nal_unit.push(payload[1]); + nal_unit.extend_from_slice(&payload[3..]); + self.fragment = Some(FragmentState { rtp_timestamp, nal_unit }); + return Ok(()); + } + + let Some(fragment) = + self.fragment.as_mut().filter(|fragment| fragment.rtp_timestamp == rtp_timestamp) + else { + // A continuation without its start means the preceding packets were lost. + self.discard_in_progress(); + return Ok(()); + }; + fragment.nal_unit.extend_from_slice(&payload[3..]); + + if end { + let nal_unit = + self.fragment.take().ok_or(RtpDepacketizerError::InvalidFragment)?.nal_unit; + self.current_mut(rtp_timestamp)?.nal_units.push(nal_unit); + } + Ok(()) + } + + /// Completes the pending H.26x access unit and queues it. + /// + /// Keyframe access units missing parameter sets receive the ones the SDP + /// carried out-of-band, so every published keyframe is self-contained: + /// passthrough subscribers may join mid-stream and can only initialize + /// their decoder from parameter sets inside the keyframe itself. + pub(super) fn finish_current(&mut self) -> Result<(), RtpDepacketizerError> { + let Some(current) = self.current.take() else { + return Ok(()); + }; + if current.nal_units.is_empty() { + return Ok(()); + } + + let mut presence = NalPresence::default(); + for nal in ¤t.nal_units { + presence.record(self.codec, nal); + } + + // Prepend only the missing parameter-set kinds, in VPS, SPS, PPS + // order; in-band parameter sets pass through untouched. + let mut nal_units = Vec::with_capacity(current.nal_units.len() + 3); + if presence.idr { + if self.codec == EncodedVideoCodec::H265 && !presence.vps { + nal_units.extend(self.parameter_sets.vps.iter().map(Vec::as_slice)); + } + if !presence.sps { + nal_units.extend(self.parameter_sets.sps.iter().map(Vec::as_slice)); + } + if !presence.pps { + nal_units.extend(self.parameter_sets.pps.iter().map(Vec::as_slice)); + } + } + nal_units.extend(current.nal_units.iter().map(Vec::as_slice)); + + let access_unit = + access_unit_from_nalus(self.codec, &nal_units, current.timestamp_us, self.resolution)?; + + if presence.idr + && access_unit.frame_type != EncodedFrameType::Key + && !self.warned_missing_parameter_sets + { + self.warned_missing_parameter_sets = true; + log::warn!( + "H.265 keyframe lacks VPS/SPS/PPS and the SDP provided none; \ + the stream cannot be published until a self-contained keyframe arrives" + ); + } + + self.enqueue(access_unit); + Ok(()) + } +} + +/// Which access-unit-defining NAL kinds appear in a pending access unit. +#[derive(Debug, Clone, Copy, Default)] +struct NalPresence { + vps: bool, + sps: bool, + pps: bool, + idr: bool, +} + +impl NalPresence { + fn record(&mut self, codec: EncodedVideoCodec, nal: &[u8]) { + let Some(&header) = nal.first() else { + return; + }; + match codec { + EncodedVideoCodec::H264 => match header & 0x1f { + 5 => self.idr = true, + 7 => self.sps = true, + 8 => self.pps = true, + _ => {} + }, + EncodedVideoCodec::H265 => match (header >> 1) & 0x3f { + 19 | 20 => self.idr = true, + 32 => self.vps = true, + 33 => self.sps = true, + 34 => self.pps = true, + _ => {} + }, + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::{ + encoded::OwnedEncodedAccessUnit, primitive::VideoResolution, + sources::rtsp::rtp::H26xParameterSets, + }; + + fn assembler_with_parameter_sets( + codec: EncodedVideoCodec, + parameter_sets: H26xParameterSets, + ) -> RtpAccessUnitAssembler { + RtpAccessUnitAssembler::new( + codec, + 96, + 90_000, + parameter_sets, + VideoResolution::new(640, 480), + ) + .unwrap() + } + + fn annex_b_nals(access_unit: &OwnedEncodedAccessUnit) -> Vec<&[u8]> { + crate::encoded::h26x::annex_b_nalus(&access_unit.payload) + } + + #[test] + fn assembles_h264_single_nal_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + } + + #[test] + fn assembles_h264_stap_a() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // STAP-A carrying SPS (2 bytes) and PPS (2 bytes), then an IDR. + let stap = rtp_packet(10, 12_000, false, &[0x18, 0, 2, 0x67, 9, 0, 2, 0x68, 8]); + let idr = rtp_packet(11, 12_000, true, &[0x65, 1]); + + assert!(push_one(&mut assembler, &stap).is_none()); + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 9, 0, 0, 0, 1, 0x68, 8, 0, 0, 0, 1, 0x65, 1] + ); + } + + #[test] + fn assembles_h264_fu_a() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x7c, 0x45, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + } + + #[test] + fn sequence_gap_recovers_h264_at_next_keyframe() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let delta = rtp_packet(12, 15_000, true, &[0x41, 1, 2]); + let key = rtp_packet(13, 18_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The gap dropped the fragment; the delta frame after it is withheld. + assert!(push_one(&mut assembler, &delta).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 1); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 3, 4]); + let stats = assembler.stats(); + assert_eq!(stats.dropped_access_units, 1); + assert!(!stats.awaiting_keyframe); + } + + #[test] + fn marker_with_open_h264_fragment_drops_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let truncated = rtp_packet(11, 12_000, true, &[0x7c, 0x05, 3, 4]); + let key = rtp_packet(12, 15_000, true, &[0x65, 5, 6]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The marker arrived without the FU end bit: the fragment is truncated. + assert!(push_one(&mut assembler, &truncated).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 0); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 5, 6]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn drops_h264_fu_continuation_without_start() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let continuation = rtp_packet(10, 12_000, false, &[0x7c, 0x05, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &continuation).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 3, 4]); + } + + #[test] + fn assembles_h265_fragment_units() { + let mut assembler = assembler(EncodedVideoCodec::H265); + // FU (type 49) carrying an IDR_W_RADL (type 19) split in two, after + // an AP (type 48) carrying VPS, SPS, and PPS. + let parameter_sets = rtp_packet( + 9, + 12_000, + false, + &[ + 0x60, 0x01, // AP NAL header. + 0, 2, 0x40, 0x01, // VPS (type 32). + 0, 2, 0x42, 0x01, // SPS (type 33). + 0, 2, 0x44, 0x01, // PPS (type 34). + ], + ); + let start = rtp_packet(10, 12_000, false, &[0x62, 0x01, 0x93, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x62, 0x01, 0x53, 3, 4]); + + assert!(push_one(&mut assembler, ¶meter_sets).is_none()); + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + let nals = annex_b_nals(&access_unit); + assert_eq!(nals.len(), 4); + assert_eq!(nals[3], &[0x26, 0x01, 1, 2, 3, 4]); + } + + #[test] + fn injects_sdp_parameter_sets_into_h264_keyframe() { + let parameter_sets = H26xParameterSets { + vps: Vec::new(), + sps: vec![vec![0x67, 9, 8]], + pps: vec![vec![0x68, 7]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H264, parameter_sets); + let idr = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + let delta = rtp_packet(11, 15_000, true, &[0x41, 3]); + + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 9, 8, 0, 0, 0, 1, 0x68, 7, 0, 0, 0, 1, 0x65, 1, 2] + ); + + // Delta frames pass through untouched. + let access_unit = push_one(&mut assembler, &delta).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn injects_sdp_parameter_sets_into_h265_keyframe() { + let parameter_sets = H26xParameterSets { + vps: vec![vec![0x40, 0x01, 1]], + sps: vec![vec![0x42, 0x01, 2]], + pps: vec![vec![0x44, 0x01, 3]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H265, parameter_sets); + // An IDR-only access unit classifies as a keyframe only once the SDP + // parameter sets are injected. + let idr = rtp_packet(10, 12_000, true, &[0x26, 0x01, 1, 2]); + + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + let nals = annex_b_nals(&access_unit); + assert_eq!(nals, vec![ + &[0x40, 0x01, 1][..], + &[0x42, 0x01, 2][..], + &[0x44, 0x01, 3][..], + &[0x26, 0x01, 1, 2][..], + ]); + } + + #[test] + fn does_not_duplicate_in_band_parameter_sets() { + let parameter_sets = H26xParameterSets { + vps: Vec::new(), + sps: vec![vec![0x67, 99]], + pps: vec![vec![0x68, 99]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H264, parameter_sets); + // The stream repeats its own parameter sets in-band. + let stap = rtp_packet(10, 12_000, false, &[0x18, 0, 2, 0x67, 1, 0, 2, 0x68, 2]); + let idr = rtp_packet(11, 12_000, true, &[0x65, 3]); + + assert!(push_one(&mut assembler, &stap).is_none()); + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x68, 2, 0, 0, 0, 1, 0x65, 3] + ); + } +} From 62cfd65a23b7139f455578f4a988713682dc9d9a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:50 -0700 Subject: [PATCH 05/38] Implement VP8/VP9 depacketization --- livekit-capture/src/sources/rtsp/rtp/vpx.rs | 464 ++++++++++++++++++++ 1 file changed, 464 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/rtp/vpx.rs diff --git a/livekit-capture/src/sources/rtsp/rtp/vpx.rs b/livekit-capture/src/sources/rtsp/rtp/vpx.rs new file mode 100644 index 000000000..28407defb --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/vpx.rs @@ -0,0 +1,464 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! VP8 (RFC 7741) and VP9 (draft-ietf-payload-vp9) RTP payload handling. +//! +//! Only single-layer streams are supported; scalable streams are rejected +//! with [`RtpDepacketizerError::UnsupportedPayloadDescriptor`]. + +use super::{RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::encoded::EncodedFrameType; + +impl RtpAccessUnitAssembler { + pub(super) fn push_vp8_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_vp8_payload_descriptor(packet.payload)?; + if descriptor.payload.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let frame = self.current_frame_mut(packet.timestamp)?; + if frame.payload.is_empty() { + if !descriptor.start_of_partition || descriptor.partition_id != 0 { + // The beginning of this frame was lost. + self.discard_in_progress(); + return Ok(()); + } + frame.frame_type = Some(if is_vp8_keyframe(descriptor.payload) { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }); + } + frame.payload.extend_from_slice(descriptor.payload); + Ok(()) + } + + pub(super) fn push_vp9_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_vp9_payload_descriptor(packet.payload)?; + if descriptor.payload.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + if descriptor.spatial_id.unwrap_or(0) != 0 + || descriptor.inter_layer_predicted.unwrap_or(false) + { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + let frame = self.current_frame_mut(packet.timestamp)?; + if frame.payload.is_empty() { + if !descriptor.beginning_of_frame { + // The beginning of this frame was lost. + self.discard_in_progress(); + return Ok(()); + } + frame.frame_type = Some( + if !descriptor.inter_picture_predicted || is_vp9_keyframe(descriptor.payload) { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }, + ); + } + frame.payload.extend_from_slice(descriptor.payload); + Ok(()) + } +} + +#[derive(Debug, Clone, Copy)] +struct Vp8PayloadDescriptor<'a> { + start_of_partition: bool, + partition_id: u8, + payload: &'a [u8], +} + +#[derive(Debug, Clone, Copy)] +struct Vp9PayloadDescriptor<'a> { + beginning_of_frame: bool, + inter_picture_predicted: bool, + spatial_id: Option, + inter_layer_predicted: Option, + payload: &'a [u8], +} + +fn parse_vp8_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let Some(&descriptor) = payload.first() else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + let start_of_partition = descriptor & 0x10 != 0; + let partition_id = descriptor & 0x0f; + let mut cursor = 1; + if descriptor & 0x80 != 0 { + let Some(&extension) = payload.get(cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + cursor += 1; + if extension & 0x80 != 0 { + let Some(&picture_id) = payload.get(cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + cursor += if picture_id & 0x80 != 0 { 2 } else { 1 }; + } + if extension & 0x40 != 0 { + cursor += 1; + } + if extension & 0x20 != 0 || extension & 0x10 != 0 { + cursor += 1; + } + } + if cursor > payload.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + Ok(Vp8PayloadDescriptor { start_of_partition, partition_id, payload: &payload[cursor..] }) +} + +fn parse_vp9_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let Some(&descriptor) = payload.first() else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if descriptor & 0x10 != 0 { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + let beginning_of_frame = descriptor & 0x08 != 0; + let inter_picture_predicted = descriptor & 0x40 != 0; + let mut cursor = 1; + if descriptor & 0x80 != 0 { + let Some(&picture_id) = payload.get(cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + cursor += if picture_id & 0x80 != 0 { 2 } else { 1 }; + } + + let mut spatial_id = None; + let mut inter_layer_predicted = None; + if descriptor & 0x20 != 0 { + let Some(&layer_info) = payload.get(cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + cursor += 1; + spatial_id = Some((layer_info >> 1) & 0x07); + inter_layer_predicted = Some(layer_info & 0x01 != 0); + cursor += 1; // TL0PICIDX is present in non-flexible mode. + } + + if descriptor & 0x02 != 0 { + skip_vp9_scalability_structure(payload, &mut cursor)?; + } + + if cursor > payload.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + Ok(Vp9PayloadDescriptor { + beginning_of_frame, + inter_picture_predicted, + spatial_id, + inter_layer_predicted, + payload: &payload[cursor..], + }) +} + +fn skip_vp9_scalability_structure( + payload: &[u8], + cursor: &mut usize, +) -> Result<(), RtpDepacketizerError> { + let Some(&structure) = payload.get(*cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + *cursor += 1; + + let spatial_layers = ((structure >> 5) & 0x07) + 1; + if spatial_layers != 1 { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + if structure & 0x10 != 0 { + let bytes = usize::from(spatial_layers) * 4; + skip_bytes(payload, cursor, bytes)?; + } + + if structure & 0x08 != 0 { + let Some(&group_count) = payload.get(*cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + *cursor += 1; + for _ in 0..group_count { + let Some(&group) = payload.get(*cursor) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + *cursor += 1; + skip_bytes(payload, cursor, usize::from((group >> 2) & 0x03))?; + } + } + + Ok(()) +} + +fn skip_bytes( + payload: &[u8], + cursor: &mut usize, + bytes: usize, +) -> Result<(), RtpDepacketizerError> { + let Some(next) = cursor.checked_add(bytes) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if next > payload.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + *cursor = next; + Ok(()) +} + +fn is_vp8_keyframe(payload: &[u8]) -> bool { + payload.first().is_some_and(|header| header & 0x01 == 0) +} + +/// Parses the start of a VP9 uncompressed frame header, whose `f(n)` fields +/// are MSB-first, and reports whether it begins a keyframe. +fn is_vp9_keyframe(payload: &[u8]) -> bool { + let Some(&first_byte) = payload.first() else { + return false; + }; + // frame_marker: f(2), must be 0b10. + if first_byte >> 6 != 0b10 { + return false; + } + + let mut bit_offset = 2usize; + let profile_low = read_bit(first_byte, bit_offset); + bit_offset += 1; + let profile_high = read_bit(first_byte, bit_offset); + bit_offset += 1; + let profile = profile_low | (profile_high << 1); + if profile == 3 { + bit_offset += 1; // reserved_zero + } + // show_existing_frame: a repeated frame is never a keyframe. + if read_bit(first_byte, bit_offset) != 0 { + return false; + } + bit_offset += 1; + // frame_type: 0 is KEY_FRAME. + read_bit(first_byte, bit_offset) == 0 +} + +/// Reads bit `bit_offset` of `byte`, counting from the most significant bit. +fn read_bit(byte: u8, bit_offset: usize) -> u8 { + (byte >> (7 - bit_offset)) & 0x01 +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::encoded::EncodedVideoCodec; + + #[test] + fn assembles_vp8_fragments() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let start = rtp_packet(10, 12_000, false, &[0x10, 0x00, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x00, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 1, 2, 3, 4]); + } + + #[test] + fn drops_vp8_mid_frame_start() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let mid_frame = rtp_packet(10, 12_000, true, &[0x00, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x10, 0x00, 3, 4]); + + assert!(push_one(&mut assembler, &mid_frame).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 3, 4]); + } + + #[test] + fn sequence_gap_recovers_vp8_at_next_keyframe() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let start = rtp_packet(10, 12_000, false, &[0x10, 0x00, 1, 2]); + let delta = rtp_packet(12, 15_000, true, &[0x10, 0x01, 3, 4]); + let key = rtp_packet(13, 18_000, true, &[0x10, 0x00, 5, 6]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The gap dropped the fragment; the delta frame after it is withheld. + assert!(push_one(&mut assembler, &delta).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 1); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 5, 6]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn assembles_vp9_single_layer_frame() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x0c, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_non_flexible_layer_descriptor() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x2c, 0x10, 7, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_single_layer_scalability_structure() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet( + 10, + 12_000, + true, + &[ + 0x0e, // B, E, V + 0x18, // one spatial layer, resolution present, picture group present + 0x01, 0x40, 0x00, 0xb4, // 320x180 + 0x01, // one picture group + 0x04, // one reference index + 0x01, // P_DIFF + 0x82, 1, 2, + ], + ); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_descriptor_keyframe_from_prediction_bit() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet( + 10, + 12_000, + true, + &[ + 0x0e, // B, E, V; P is clear, so this is not inter-picture predicted. + 0x18, // one spatial layer, resolution present, picture group present + 0x02, 0x80, 0x01, 0x68, // 640x360 + 0x01, // one picture group + 0x04, // one reference index + 0x01, // P_DIFF + 0xb1, 1, 2, + ], + ); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0xb1, 1, 2]); + } + + #[test] + fn assembles_vp9_predicted_frame_as_delta() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + // P is set and the payload is an inter frame: must not classify as Key. + let packet = rtp_packet(10, 12_000, true, &[0x4c, 0x86, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x86, 1, 2]); + } + + #[test] + fn vp9_bitstream_keyframe_overrides_predicted_bit() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + // P is set but the uncompressed header says KEY_FRAME. + let packet = rtp_packet(10, 12_000, true, &[0x4c, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn classifies_vp9_uncompressed_header_frame_types() { + // 0b1000_0010: marker, profile 0, show_existing=0, KEY_FRAME, show_frame=1. + assert!(is_vp9_keyframe(&[0x82])); + // 0b1000_0011: keyframe with error_resilient_mode set. + assert!(is_vp9_keyframe(&[0x83])); + // 0b1011_0000: profile 3 keyframe. + assert!(is_vp9_keyframe(&[0xb0])); + // 0b1000_0110: frame_type=1, an inter frame. + assert!(!is_vp9_keyframe(&[0x86])); + // 0b1011_0010: profile 3 inter frame. + assert!(!is_vp9_keyframe(&[0xb2])); + // 0b1000_1000: show_existing_frame repeats a decoded frame. + assert!(!is_vp9_keyframe(&[0x88])); + // 0b0000_0010: invalid frame_marker. + assert!(!is_vp9_keyframe(&[0x02])); + assert!(!is_vp9_keyframe(&[])); + } + + #[test] + fn rejects_vp9_multi_layer_scalability_structure() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x0e, 0x20, 0x82, 1, 2]); + + let err = assembler.push(&packet).unwrap_err(); + assert_eq!(err, RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + #[test] + fn drops_vp9_mid_frame_start() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let mid_frame = rtp_packet(10, 12_000, true, &[0x04, 0x82, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x0c, 0x82, 3, 4]); + + assert!(push_one(&mut assembler, &mid_frame).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 3, 4]); + } + + #[test] + fn rejects_vp9_flexible_mode() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x1c, 0xa2, 1, 2]); + + let err = assembler.push(&packet).unwrap_err(); + assert_eq!(err, RtpDepacketizerError::UnsupportedPayloadDescriptor); + } +} From 559d0143f7700ff988ab36d992b7b060be85b223 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:50 -0700 Subject: [PATCH 06/38] Implement AV1 depacketization --- livekit-capture/src/sources/rtsp/rtp/av1.rs | 339 ++++++++++++++++++++ 1 file changed, 339 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/rtp/av1.rs diff --git a/livekit-capture/src/sources/rtsp/rtp/av1.rs b/livekit-capture/src/sources/rtsp/rtp/av1.rs new file mode 100644 index 000000000..54b2e4541 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/av1.rs @@ -0,0 +1,339 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AV1 RTP payload handling (aomediacodec RTP payload specification). +//! +//! RTP carries OBU elements without size fields; assembly re-frames them as +//! size-prefixed OBUs so the access unit is a low-overhead AV1 bitstream. + +use super::{Av1FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::{ + encoded::EncodedFrameType, + sources::rtsp::bits::{read_leb128, write_leb128, BitReader}, +}; + +impl RtpAccessUnitAssembler { + pub(super) fn push_av1_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_av1_payload_descriptor(packet.payload)?; + if descriptor.elements.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let last_index = descriptor.elements.len() - 1; + for (index, element) in descriptor.elements.iter().enumerate() { + if element.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let obu = if index == 0 && descriptor.starts_fragment { + let Some(fragment) = self + .av1_fragment + .take() + .filter(|fragment| fragment.rtp_timestamp == packet.timestamp) + else { + // A continuation without its start means the preceding packets were lost. + self.discard_in_progress(); + return Ok(()); + }; + let mut obu = fragment.obu; + obu.extend_from_slice(element); + obu + } else { + if index == 0 && self.av1_fragment.is_some() { + return Err(RtpDepacketizerError::InvalidFragment); + } + element.to_vec() + }; + + if index == last_index && descriptor.ends_fragment { + self.av1_fragment = Some(Av1FragmentState { rtp_timestamp: packet.timestamp, obu }); + return Ok(()); + } + + let mut obu = av1_obu_from_rtp_element(&obu)?; + let frame = self.current_frame_mut(packet.timestamp)?; + if let Some(reduced_still_picture_header) = av1_reduced_still_picture_header(&obu)? { + frame.av1_reduced_still_picture_header = Some(reduced_still_picture_header); + } + if frame.frame_type.is_none() { + frame.frame_type = av1_frame_type(&obu, frame.av1_reduced_still_picture_header)?; + } + frame.payload.append(&mut obu); + } + + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct Av1PayloadDescriptor<'a> { + starts_fragment: bool, + ends_fragment: bool, + elements: Vec<&'a [u8]>, +} + +fn parse_av1_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let Some(&header) = payload.first() else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + let starts_fragment = header & 0x80 != 0; + let ends_fragment = header & 0x40 != 0; + let element_count = (header >> 4) & 0x03; + + let mut cursor = 1; + let mut elements = Vec::new(); + if element_count == 0 { + while cursor < payload.len() { + let len = read_leb128(payload, &mut cursor) + .ok_or(RtpDepacketizerError::UnsupportedPayload)?; + let Some(end) = cursor.checked_add(len) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if end > payload.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + elements.push(&payload[cursor..end]); + cursor = end; + } + } else { + for index in 0..usize::from(element_count) { + let len = if index + 1 == usize::from(element_count) { + payload.len().saturating_sub(cursor) + } else { + read_leb128(payload, &mut cursor) + .ok_or(RtpDepacketizerError::UnsupportedPayload)? + }; + let Some(end) = cursor.checked_add(len) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if end > payload.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + elements.push(&payload[cursor..end]); + cursor = end; + } + } + + Ok(Av1PayloadDescriptor { starts_fragment, ends_fragment, elements }) +} + +/// Converts an RTP OBU element into a size-prefixed OBU. +fn av1_obu_from_rtp_element(element: &[u8]) -> Result, RtpDepacketizerError> { + let Some(&header) = element.first() else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if header & 0x80 != 0 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + if header & 0x02 != 0 { + let mut cursor = if header & 0x04 != 0 { 2 } else { 1 }; + if cursor > element.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + let payload_size = + read_leb128(element, &mut cursor).ok_or(RtpDepacketizerError::UnsupportedPayload)?; + if payload_size != element.len().saturating_sub(cursor) { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + return Ok(element.to_vec()); + } + + let payload_offset = if header & 0x04 != 0 { 2 } else { 1 }; + if payload_offset > element.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let payload_size = element.len() - payload_offset; + let mut obu = Vec::with_capacity(element.len() + 8); + obu.push(header | 0x02); + if header & 0x04 != 0 { + obu.push(element[1]); + } + write_leb128(payload_size, &mut obu); + obu.extend_from_slice(&element[payload_offset..]); + Ok(obu) +} + +/// Reads `reduced_still_picture_header` from a sequence header OBU. +fn av1_reduced_still_picture_header(obu: &[u8]) -> Result, RtpDepacketizerError> { + let Some((obu_type, payload)) = av1_obu_parts(obu)? else { + return Ok(None); + }; + if obu_type != 1 { + return Ok(None); + } + + let mut reader = BitReader::new(payload); + reader.read_bits(3).ok_or(RtpDepacketizerError::UnsupportedPayload)?; // seq_profile + reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)?; // still_picture + Ok(Some(reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)? != 0)) +} + +/// Classifies a frame or frame-header OBU, when `obu` is one. +fn av1_frame_type( + obu: &[u8], + reduced_still_picture_header: Option, +) -> Result, RtpDepacketizerError> { + let Some((obu_type, payload)) = av1_obu_parts(obu)? else { + return Ok(None); + }; + if !matches!(obu_type, 3 | 6) { + return Ok(None); + } + + if reduced_still_picture_header.unwrap_or(false) { + return Ok(Some(EncodedFrameType::Key)); + } + + let mut reader = BitReader::new(payload); + let show_existing_frame = reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)?; + if show_existing_frame != 0 { + return Ok(Some(EncodedFrameType::Delta)); + } + + let frame_type = reader.read_bits(2).ok_or(RtpDepacketizerError::UnsupportedPayload)?; + Ok(Some(if frame_type == 0 { EncodedFrameType::Key } else { EncodedFrameType::Delta })) +} + +/// Splits an OBU into its type and payload bytes. +fn av1_obu_parts(obu: &[u8]) -> Result, RtpDepacketizerError> { + let Some(&header) = obu.first() else { + return Ok(None); + }; + if header & 0x80 != 0 { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let obu_type = (header & 0x78) >> 3; + let has_extension = header & 0x04 != 0; + let has_size = header & 0x02 != 0; + let mut cursor = if has_extension { 2 } else { 1 }; + if cursor > obu.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + if !has_size { + return Ok(Some((obu_type, &obu[cursor..]))); + } + + let payload_size = + read_leb128(obu, &mut cursor).ok_or(RtpDepacketizerError::UnsupportedPayload)?; + let Some(end) = cursor.checked_add(payload_size) else { + return Err(RtpDepacketizerError::UnsupportedPayload); + }; + if end > obu.len() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + Ok(Some((obu_type, &obu[cursor..end]))) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::encoded::EncodedVideoCodec; + + fn av1_sequence_and_frame_rtp_payload(frame_header: u8) -> [u8; 6] { + [ + 0x28, // W=2, N=1. + 0x02, // First OBU element length. + 0x08, // Sequence header OBU without the size field. + 0x00, // profile=0, still_picture=false, reduced_still_picture_header=false. + 0x30, // Frame OBU without the size field. + frame_header, + ] + } + + #[test] + fn assembles_av1_temporal_unit() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::AV1); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + } + + #[test] + fn av1_sequence_header_before_inter_frame_is_delta() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &av1_sequence_and_frame_rtp_payload(0x38)); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x38]); + } + + #[test] + fn assembles_fragmented_av1_obu() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let start = rtp_packet(10, 12_000, false, &[0x50, 0x30, 0x38]); + let end = rtp_packet(11, 12_000, true, &[0x90, 2, 3]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x32, 0x03, 0x38, 2, 3]); + } + + #[test] + fn assembles_av1_obu_payload_with_size_field() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &[0x10, 0x30, 0x38, 2, 3]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x32, 0x03, 0x38, 2, 3]); + } + + #[test] + fn marker_with_open_av1_fragment_drops_frame() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + // Y is set, so the OBU fragment is unterminated when the marker closes it. + let truncated = rtp_packet(10, 12_000, true, &[0x50, 0x30, 1]); + let key = rtp_packet(11, 15_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + assert!(push_one(&mut assembler, &truncated).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn drops_av1_fragment_continuation_without_start() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + // Z is set: this continues an OBU whose start was never received. + let continuation = rtp_packet(10, 12_000, true, &[0x90, 2, 3]); + let key = rtp_packet(11, 15_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + assert!(push_one(&mut assembler, &continuation).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + } +} From f229a9752be24f19458ce2c2b48ced2fb963e2b4 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:51 -0700 Subject: [PATCH 07/38] Implement authentication --- livekit-capture/src/sources/rtsp/auth.rs | 401 +++++++++++++++++++++++ 1 file changed, 401 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/auth.rs diff --git a/livekit-capture/src/sources/rtsp/auth.rs b/livekit-capture/src/sources/rtsp/auth.rs new file mode 100644 index 000000000..82804185d --- /dev/null +++ b/livekit-capture/src/sources/rtsp/auth.rs @@ -0,0 +1,401 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP authentication: Basic and Digest (MD5 with `qop=auth`). + +use std::{ + collections::hash_map::RandomState, + fmt, + hash::{BuildHasher, Hasher}, +}; + +use base64::{engine::general_purpose, Engine as _}; +use md5::{Digest, Md5}; + +use super::{client::RtspResponse, RtspVideoSourceError}; + +/// Username and password for RTSP authentication. +#[derive(Clone, PartialEq, Eq)] +pub(super) struct RtspCredentials { + pub(super) username: String, + pub(super) password: String, +} + +// Manual so a `{:?}` of the source or its context never prints the password. +impl fmt::Debug for RtspCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspCredentials") + .field("username", &self.username) + .field("password", &"") + .finish() + } +} + +/// Tracks the server's authentication challenge across requests. +#[derive(Debug, Clone)] +pub(super) struct RtspAuthContext { + credentials: Option, + challenge: Option, + nonce_count: u32, + cnonce: String, +} + +impl RtspAuthContext { + /// Creates an authentication context; without credentials, a challenge + /// fails with [`RtspVideoSourceError::MissingCredentials`]. + pub(super) fn new(credentials: Option) -> Self { + Self { credentials, challenge: None, nonce_count: 0, cnonce: make_cnonce() } + } + + /// Builds the `Authorization` header value for a request, once the server + /// has issued a challenge. + pub(super) fn header( + &mut self, + method: &str, + uri: &str, + ) -> Result, RtspVideoSourceError> { + let Some(challenge) = self.challenge.clone() else { + return Ok(None); + }; + let credentials = + self.credentials.as_ref().ok_or(RtspVideoSourceError::MissingCredentials)?; + match challenge { + RtspAuthChallenge::Basic => { + let token = general_purpose::STANDARD + .encode(format!("{}:{}", credentials.username, credentials.password)); + Ok(Some(format!("Basic {token}"))) + } + RtspAuthChallenge::Digest(challenge) => { + self.nonce_count = self.nonce_count.saturating_add(1); + Ok(Some(build_digest_authorization( + credentials, + &challenge, + method, + uri, + self.nonce_count, + &self.cnonce, + ))) + } + } + } + + /// Ingests the challenge of a 401 response so the retry can authenticate. + pub(super) fn update_from_unauthorized( + &mut self, + response: &RtspResponse, + ) -> Result<(), RtspVideoSourceError> { + if self.credentials.is_none() { + return Err(RtspVideoSourceError::MissingCredentials); + } + self.challenge = Some(parse_authenticate_header( + response.headers("www-authenticate").collect::>().as_slice(), + )?); + self.nonce_count = 0; + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum RtspAuthChallenge { + Basic, + Digest(DigestAuthChallenge), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct DigestAuthChallenge { + realm: String, + nonce: String, + opaque: Option, + qop: Option, +} + +fn parse_authenticate_header( + headers: &[&str], +) -> Result { + for header in headers { + if strip_auth_scheme(header, "Digest").is_some() { + return parse_digest_challenge(header); + } + } + for header in headers { + if strip_auth_scheme(header, "Basic").is_some() { + return Ok(RtspAuthChallenge::Basic); + } + } + let scheme = headers + .first() + .and_then(|header| header.split_whitespace().next()) + .unwrap_or_default() + .to_owned(); + Err(RtspVideoSourceError::UnsupportedAuthScheme(scheme)) +} + +fn parse_digest_challenge(header: &str) -> Result { + let params = parse_auth_params( + strip_auth_scheme(header, "Digest").ok_or(RtspVideoSourceError::InvalidAuthChallenge)?, + ); + let realm = params + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("realm")) + .map(|(_, value)| value.to_owned()) + .ok_or(RtspVideoSourceError::InvalidAuthChallenge)?; + let nonce = params + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("nonce")) + .map(|(_, value)| value.to_owned()) + .ok_or(RtspVideoSourceError::InvalidAuthChallenge)?; + if let Some((_, algorithm)) = + params.iter().find(|(name, _)| name.eq_ignore_ascii_case("algorithm")) + { + if !algorithm.eq_ignore_ascii_case("MD5") { + return Err(RtspVideoSourceError::UnsupportedDigestAlgorithm(algorithm.clone())); + } + } + let qop = params + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("qop")) + .and_then(|(_, value)| select_digest_qop(value)); + let opaque = params + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("opaque")) + .map(|(_, value)| value.to_owned()); + + Ok(RtspAuthChallenge::Digest(DigestAuthChallenge { realm, nonce, opaque, qop })) +} + +fn strip_auth_scheme<'a>(header: &'a str, scheme: &str) -> Option<&'a str> { + let header = header.trim_start(); + let rest = header.get(scheme.len()..)?; + if !header[..scheme.len()].eq_ignore_ascii_case(scheme) { + return None; + } + if rest.is_empty() { + return Some(rest); + } + rest.strip_prefix(' ') +} + +fn parse_auth_params(params: &str) -> Vec<(String, String)> { + let mut parsed = Vec::new(); + let mut current = String::new(); + let mut in_quotes = false; + let mut escaped = false; + for ch in params.chars() { + if escaped { + current.push(ch); + escaped = false; + continue; + } + match ch { + '\\' if in_quotes => { + escaped = true; + current.push(ch); + } + '"' => { + in_quotes = !in_quotes; + current.push(ch); + } + ',' if !in_quotes => { + push_auth_param(&mut parsed, ¤t); + current.clear(); + } + _ => current.push(ch), + } + } + push_auth_param(&mut parsed, ¤t); + parsed +} + +fn push_auth_param(parsed: &mut Vec<(String, String)>, param: &str) { + let Some((name, value)) = param.trim().split_once('=') else { + return; + }; + parsed.push((name.trim().to_owned(), unquote_auth_value(value.trim()))); +} + +fn unquote_auth_value(value: &str) -> String { + let Some(value) = value.strip_prefix('"').and_then(|value| value.strip_suffix('"')) else { + return value.to_owned(); + }; + let mut unquoted = String::new(); + let mut escaped = false; + for ch in value.chars() { + if escaped { + unquoted.push(ch); + escaped = false; + } else if ch == '\\' { + escaped = true; + } else { + unquoted.push(ch); + } + } + unquoted +} + +fn select_digest_qop(value: &str) -> Option { + value.split(',').map(str::trim).find(|qop| qop.eq_ignore_ascii_case("auth")).map(str::to_owned) +} + +fn build_digest_authorization( + credentials: &RtspCredentials, + challenge: &DigestAuthChallenge, + method: &str, + uri: &str, + nonce_count: u32, + cnonce: &str, +) -> String { + let ha1 = + md5_hex(format!("{}:{}:{}", credentials.username, challenge.realm, credentials.password)); + let ha2 = md5_hex(format!("{method}:{uri}")); + let response = if let Some(qop) = &challenge.qop { + md5_hex(format!("{ha1}:{}:{nonce_count:08x}:{cnonce}:{qop}:{ha2}", challenge.nonce)) + } else { + md5_hex(format!("{ha1}:{}:{ha2}", challenge.nonce)) + }; + + let mut header = format!( + "Digest username=\"{}\", realm=\"{}\", nonce=\"{}\", uri=\"{}\", response=\"{}\"", + quote_auth_value(&credentials.username), + quote_auth_value(&challenge.realm), + quote_auth_value(&challenge.nonce), + quote_auth_value(uri), + response + ); + if let Some(qop) = &challenge.qop { + header.push_str(&format!( + ", qop={}, nc={nonce_count:08x}, cnonce=\"{}\"", + quote_auth_value(qop), + quote_auth_value(cnonce) + )); + } + if let Some(opaque) = &challenge.opaque { + header.push_str(&format!(", opaque=\"{}\"", quote_auth_value(opaque))); + } + header +} + +fn quote_auth_value(value: &str) -> String { + value.replace('\\', "\\\\").replace('"', "\\\"") +} + +fn md5_hex(input: impl AsRef<[u8]>) -> String { + format!("{:x}", Md5::digest(input)) +} + +/// Builds an unpredictable client nonce. Each [`RandomState`] draws fresh +/// OS-seeded keys, so chaining two independent states yields 128 bits of +/// entropy without adding an RNG dependency. +fn make_cnonce() -> String { + let mut hasher = RandomState::new().build_hasher(); + hasher.write_u64(0x6c6b_7274_7370); + let high = hasher.finish(); + let mut hasher = RandomState::new().build_hasher(); + hasher.write_u64(high); + let low = hasher.finish(); + format!("{high:016x}{low:016x}") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn builds_digest_authorization_with_qop_auth() { + // RFC 2617 section 3.5 example values. + let credentials = RtspCredentials { + username: "Mufasa".to_owned(), + password: "Circle Of Life".to_owned(), + }; + let challenge = DigestAuthChallenge { + realm: "testrealm@host.com".to_owned(), + nonce: "dcd98b7102dd2f0e8b11d0f600bfb0c093".to_owned(), + opaque: Some("5ccc069c403ebaf9f0171e9517f40e41".to_owned()), + qop: Some("auth".to_owned()), + }; + + let authorization = build_digest_authorization( + &credentials, + &challenge, + "GET", + "/dir/index.html", + 1, + "0a4f113b", + ); + + assert!(authorization.contains("response=\"6629fae49393a05397450978507c4ef1\"")); + assert!(authorization.contains("qop=auth")); + assert!(authorization.contains("nc=00000001")); + assert!(authorization.contains("opaque=\"5ccc069c403ebaf9f0171e9517f40e41\"")); + } + + #[test] + fn parses_digest_challenge_with_quoted_values() { + let challenge = parse_authenticate_header(&[ + "Digest realm=\"a, \\\"quoted\\\" realm\", nonce=\"abc\", qop=\"auth,auth-int\"", + ]) + .unwrap(); + + assert_eq!( + challenge, + RtspAuthChallenge::Digest(DigestAuthChallenge { + realm: "a, \"quoted\" realm".to_owned(), + nonce: "abc".to_owned(), + opaque: None, + qop: Some("auth".to_owned()), + }) + ); + } + + #[test] + fn prefers_digest_over_basic() { + let challenge = parse_authenticate_header(&[ + "Basic realm=\"camera\"", + "Digest realm=\"camera\", nonce=\"abc\"", + ]) + .unwrap(); + assert!(matches!(challenge, RtspAuthChallenge::Digest(_))); + } + + #[test] + fn rejects_unsupported_auth_scheme() { + let err = parse_authenticate_header(&["Bearer token=\"abc\""]).unwrap_err(); + match err { + RtspVideoSourceError::UnsupportedAuthScheme(scheme) => assert_eq!(scheme, "Bearer"), + other => panic!("expected unsupported auth scheme, got {other:?}"), + } + } + + #[test] + fn rejects_unsupported_digest_algorithm() { + let err = parse_authenticate_header(&[ + "Digest realm=\"camera\", nonce=\"abc\", algorithm=SHA-256", + ]) + .unwrap_err(); + assert!(matches!(err, RtspVideoSourceError::UnsupportedDigestAlgorithm(algorithm) if algorithm == "SHA-256")); + } + + #[test] + fn debug_redacts_password() { + let credentials = + RtspCredentials { username: "admin".to_owned(), password: "secret".to_owned() }; + let debug = format!("{credentials:?}"); + assert!(debug.contains("admin")); + assert!(!debug.contains("secret")); + } + + #[test] + fn cnonces_are_distinct() { + assert_ne!(make_cnonce(), make_cnonce()); + } +} From fe5440ed14e2fd39308b8b4e4c9f23fc4dc3ff53 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:51 -0700 Subject: [PATCH 08/38] Implement RTSP client --- livekit-capture/src/sources/rtsp/client.rs | 656 +++++++++++++++++++++ 1 file changed, 656 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/client.rs diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs new file mode 100644 index 000000000..47e309de4 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -0,0 +1,656 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP/1.0 client over TCP: request writing, response reading, and the +//! interleaved (`$`-framed) stream demuxer. + +use std::{ + fmt, + io::{self, Read, Write}, + net::{TcpStream, ToSocketAddrs}, + str, + time::{Duration, Instant}, +}; + +use bytes::{Buf, Bytes, BytesMut}; + +use super::{auth::RtspAuthContext, auth::RtspCredentials, RtspPhase, RtspVideoSourceError}; + +/// Socket read timeout: the poll granularity for the stop token, keepalives, +/// and deadlines, kept near one frame interval per the pump contract. +const READ_POLL: Duration = Duration::from_millis(100); + +/// Socket write timeout; requests are small, so a stalled write means the +/// connection is gone. +const WRITE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Upper bound on an RTSP response header. +const MAX_HEADER_BYTES: usize = 64 * 1024; + +/// Bytes requested from the socket per read. +const READ_CHUNK_BYTES: usize = 8 * 1024; + +/// A parsed `rtsp://` URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RtspUrl { + /// Request URI with any userinfo stripped, so credentials never appear + /// on the wire outside the `Authorization` header. + pub(super) request_uri: String, + /// Value for the `Host` header, always including the port. + pub(super) host_header: String, + /// Credentials from the URL userinfo, percent-decoded. + pub(super) credentials: Option, + connect_host: String, + port: u16, +} + +impl RtspUrl { + /// Parses an `rtsp://[user:password@]host[:port][/path]` URL. + pub(super) fn parse(url: &str) -> Result { + let Some(rest) = url.strip_prefix("rtsp://") else { + return Err(RtspVideoSourceError::InvalidUrl("expected rtsp:// scheme")); + }; + let (authority, path_suffix) = match rest.find('/') { + Some(path_start) => (&rest[..path_start], &rest[path_start..]), + None => (rest, ""), + }; + + let (credentials, host_port) = match authority.rsplit_once('@') { + Some((userinfo, host_port)) => (Some(parse_userinfo(userinfo)?), host_port), + None => (None, authority), + }; + if host_port.is_empty() { + return Err(RtspVideoSourceError::InvalidUrl("missing host")); + } + let (connect_host, port) = parse_host_port(host_port)?; + let host_header = if host_port.contains(':') { + host_port.to_owned() + } else { + format!("{host_port}:{port}") + }; + + Ok(Self { + request_uri: format!("rtsp://{host_port}{path_suffix}"), + host_header, + credentials, + connect_host, + port, + }) + } +} + +fn parse_userinfo(userinfo: &str) -> Result { + let (username, password) = userinfo.split_once(':').unwrap_or((userinfo, "")); + if username.is_empty() { + return Err(RtspVideoSourceError::InvalidUrl("missing username")); + } + Ok(RtspCredentials { + username: percent_decode(username), + password: percent_decode(password), + }) +} + +/// Decodes RFC 3986 percent-escapes; malformed escapes pass through as-is. +fn percent_decode(value: &str) -> String { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut cursor = 0; + while cursor < bytes.len() { + let escape = (bytes[cursor] == b'%') + .then(|| bytes.get(cursor + 1..cursor + 3)) + .flatten() + .and_then(|digits| str::from_utf8(digits).ok()) + .and_then(|digits| u8::from_str_radix(digits, 16).ok()); + if let Some(byte) = escape { + decoded.push(byte); + cursor += 3; + } else { + decoded.push(bytes[cursor]); + cursor += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +fn parse_host_port(host_port: &str) -> Result<(String, u16), RtspVideoSourceError> { + if let Some(rest) = host_port.strip_prefix('[') { + let Some((host, after_host)) = rest.split_once(']') else { + return Err(RtspVideoSourceError::InvalidUrl("malformed IPv6 host")); + }; + let port = after_host.strip_prefix(':').map(parse_port).transpose()?.unwrap_or(554); + return Ok((host.to_owned(), port)); + } + + if let Some((host, port)) = host_port.rsplit_once(':') { + if !host.contains(':') { + return Ok((host.to_owned(), parse_port(port)?)); + } + } + + Ok((host_port.to_owned(), 554)) +} + +fn parse_port(port: &str) -> Result { + port.parse().map_err(|_| RtspVideoSourceError::InvalidUrl("invalid port")) +} + +/// A parsed RTSP response. +#[derive(Debug, Clone)] +pub(super) struct RtspResponse { + pub(super) status_code: u16, + pub(super) reason: String, + headers: Vec<(String, String)>, + pub(super) body: Vec, +} + +impl RtspResponse { + pub(super) fn is_success(&self) -> bool { + (200..300).contains(&self.status_code) + } + + /// Returns the first header with the given name, case-insensitively. + pub(super) fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// Returns every header with the given name, case-insensitively. + pub(super) fn headers<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.headers + .iter() + .filter(move |(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +/// One unit read from the interleaved stream. +#[derive(Debug)] +pub(super) enum InterleavedPoll { + /// An interleaved binary frame. + Frame { + /// Interleaved channel the frame arrived on. + channel: u8, + /// Frame payload, without the 4-byte interleaved header. + payload: Bytes, + }, + /// An in-stream RTSP response, such as a keepalive reply. + Response(RtspResponse), + /// A read timed out; framing state is preserved for the next poll. + TimedOut, + /// The stream ended cleanly at a unit boundary. + EndOfStream, +} + +/// Result of one attempt to read more stream bytes. +enum StreamFill { + Filled, + Eof, + TimedOut, +} + +/// RTSP connection: owns the TCP stream, the read buffer, the request +/// sequence number, and the authentication context. +pub(super) struct RtspClient { + stream: TcpStream, + buf: BytesMut, + scratch: Vec, + cseq: u32, + auth: RtspAuthContext, + host_header: String, + last_read_at: Instant, +} + +// Manual so the read buffer's contents are not dumped; the authentication +// context redacts its own credentials. +impl fmt::Debug for RtspClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspClient") + .field("host_header", &self.host_header) + .field("cseq", &self.cseq) + .finish_non_exhaustive() + } +} + +impl RtspClient { + /// Connects to the URL's host, bounded by `deadline`, and prepares the + /// socket for polled reads. + pub(super) fn connect( + url: &RtspUrl, + credentials: Option, + deadline: Instant, + ) -> Result { + let addrs = (url.connect_host.as_str(), url.port).to_socket_addrs()?; + let mut last_error = None; + let mut stream = None; + for addr in addrs { + let Some(remaining) = deadline.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()) else { + break; + }; + match TcpStream::connect_timeout(&addr, remaining) { + Ok(connected) => { + stream = Some(connected); + break; + } + Err(err) => last_error = Some(err), + } + } + let Some(stream) = stream else { + return Err(match last_error { + Some(err) => RtspVideoSourceError::Io(err), + None => RtspVideoSourceError::Timeout { phase: RtspPhase::Connect }, + }); + }; + + if let Err(err) = stream.set_nodelay(true) { + log::debug!("failed to disable Nagle's algorithm on the RTSP stream: {err}"); + } + stream.set_read_timeout(Some(READ_POLL))?; + stream.set_write_timeout(Some(WRITE_TIMEOUT))?; + + Ok(Self { + stream, + buf: BytesMut::with_capacity(READ_CHUNK_BYTES), + scratch: vec![0; READ_CHUNK_BYTES], + cseq: 1, + auth: RtspAuthContext::new(credentials), + host_header: url.host_header.clone(), + last_read_at: Instant::now(), + }) + } + + /// Sends a request and reads its response, bounded by `deadline`, + /// retrying once with credentials on a 401 challenge. Non-2xx statuses + /// become [`RtspVideoSourceError::RtspStatus`]. + pub(super) fn request( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, &str)], + deadline: Instant, + phase: RtspPhase, + ) -> Result { + self.write_request(method, uri, headers)?; + let mut response = self.read_response(deadline, phase)?; + if response.status_code == 401 { + self.auth.update_from_unauthorized(&response)?; + self.write_request(method, uri, headers)?; + response = self.read_response(deadline, phase)?; + } + + if !response.is_success() { + return Err(RtspVideoSourceError::RtspStatus { + code: response.status_code, + reason: response.reason, + }); + } + Ok(response) + } + + /// Writes a request as a single buffered write, without waiting for the + /// response. Used for keepalives and TEARDOWN, whose replies (if any) + /// arrive in-band. + pub(super) fn write_request( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, &str)], + ) -> Result<(), RtspVideoSourceError> { + use fmt::Write as _; + + let cseq = self.cseq; + self.cseq = self.cseq.saturating_add(1); + let authorization = self.auth.header(method, uri)?; + + let mut request = String::with_capacity(256); + // Writing to a `String` cannot fail. + let _ = write!(request, "{method} {uri} RTSP/1.0\r\n"); + let _ = write!(request, "CSeq: {cseq}\r\n"); + let _ = write!(request, "User-Agent: livekit-capture/0.1\r\n"); + let _ = write!(request, "Host: {}\r\n", self.host_header); + if let Some(authorization) = authorization { + let _ = write!(request, "Authorization: {authorization}\r\n"); + } + for (name, value) in headers { + let _ = write!(request, "{name}: {value}\r\n"); + } + request.push_str("\r\n"); + + self.stream.write_all(request.as_bytes())?; + self.stream.flush()?; + Ok(()) + } + + /// Reads one RTSP response from the stream, bounded by `deadline`. + fn read_response( + &mut self, + deadline: Instant, + phase: RtspPhase, + ) -> Result { + loop { + if let Some((response, consumed)) = parse_response(&self.buf)? { + self.buf.advance(consumed); + return Ok(response); + } + match self.fill()? { + StreamFill::Filled => {} + StreamFill::Eof => { + return Err(io::Error::from(io::ErrorKind::UnexpectedEof).into()); + } + StreamFill::TimedOut => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::Timeout { phase }); + } + } + } + } + } + + /// Reads the next interleaved unit, returning within roughly one + /// [`READ_POLL`] when the stream is silent. Framing state survives + /// timed-out reads. + pub(super) fn poll_unit(&mut self) -> Result { + loop { + if let Some(unit) = self.parse_front()? { + return Ok(unit); + } + match self.fill()? { + StreamFill::Filled => {} + StreamFill::Eof => { + if self.buf.is_empty() { + return Ok(InterleavedPoll::EndOfStream); + } + // The stream ended inside an interleaved unit. + return Err(io::Error::from(io::ErrorKind::UnexpectedEof).into()); + } + StreamFill::TimedOut => return Ok(InterleavedPoll::TimedOut), + } + } + } + + /// Time since stream bytes last arrived, for the idle limit. + pub(super) fn idle_for(&self) -> Duration { + self.last_read_at.elapsed() + } + + /// Parses one complete unit from the front of the buffer. + fn parse_front(&mut self) -> Result, RtspVideoSourceError> { + let Some(&magic) = self.buf.first() else { + return Ok(None); + }; + match magic { + b'$' => { + if self.buf.len() < 4 { + return Ok(None); + } + let channel = self.buf[1]; + let payload_len = u16::from_be_bytes([self.buf[2], self.buf[3]]) as usize; + if self.buf.len() < 4 + payload_len { + return Ok(None); + } + self.buf.advance(4); + let payload = self.buf.split_to(payload_len).freeze(); + Ok(Some(InterleavedPoll::Frame { channel, payload })) + } + b'R' => match parse_response(&self.buf)? { + Some((response, consumed)) => { + self.buf.advance(consumed); + Ok(Some(InterleavedPoll::Response(response))) + } + None => Ok(None), + }, + _ => Err(RtspVideoSourceError::UnexpectedData), + } + } + + /// Reads more stream bytes into the buffer. + fn fill(&mut self) -> Result { + loop { + match self.stream.read(&mut self.scratch) { + Ok(0) => return Ok(StreamFill::Eof), + Ok(read) => { + self.buf.extend_from_slice(&self.scratch[..read]); + self.last_read_at = Instant::now(); + return Ok(StreamFill::Filled); + } + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) if is_timeout_io_error(&err) => return Ok(StreamFill::TimedOut), + Err(err) => return Err(err.into()), + } + } + } +} + +fn is_timeout_io_error(err: &io::Error) -> bool { + matches!(err.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) +} + +/// Parses one RTSP response from the front of `buf`, returning the response +/// and the bytes it consumed, or `Ok(None)` when more bytes are needed. +fn parse_response(buf: &[u8]) -> Result, RtspVideoSourceError> { + let Some(header_end) = find_header_end(buf) else { + if buf.len() > MAX_HEADER_BYTES { + return Err(RtspVideoSourceError::InvalidResponse("header too large")); + } + return Ok(None); + }; + + let header_text = str::from_utf8(&buf[..header_end]) + .map_err(|_| RtspVideoSourceError::InvalidResponse("header is not UTF-8"))?; + let mut lines = header_text.split("\r\n"); + let status_line = + lines.next().ok_or(RtspVideoSourceError::InvalidResponse("missing status line"))?; + let mut status_parts = status_line.splitn(3, ' '); + if status_parts.next() != Some("RTSP/1.0") { + return Err(RtspVideoSourceError::InvalidResponse("unsupported version")); + } + let status_code = status_parts + .next() + .ok_or(RtspVideoSourceError::InvalidResponse("missing status code"))? + .parse() + .map_err(|_| RtspVideoSourceError::InvalidResponse("invalid status code"))?; + let reason = status_parts.next().unwrap_or_default().to_owned(); + + let mut headers = Vec::new(); + for line in lines { + let Some((name, value)) = line.split_once(':') else { + return Err(RtspVideoSourceError::InvalidResponse("malformed header")); + }; + headers.push((name.trim().to_owned(), value.trim().to_owned())); + } + + let content_length = headers + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) + .map(|(_, value)| value.parse::()) + .transpose() + .map_err(|_| RtspVideoSourceError::InvalidResponse("invalid content length"))? + .unwrap_or(0); + let body_start = header_end + 4; + let Some(consumed) = body_start.checked_add(content_length) else { + return Err(RtspVideoSourceError::InvalidResponse("invalid content length")); + }; + if buf.len() < consumed { + return Ok(None); + } + let body = buf[body_start..consumed].to_vec(); + + Ok(Some((RtspResponse { status_code, reason, headers, body }, consumed))) +} + +/// Finds the end of the response header (the start of `\r\n\r\n`). +fn find_header_end(buf: &[u8]) -> Option { + buf.windows(4).take(MAX_HEADER_BYTES).position(|window| window == b"\r\n\r\n") +} + +/// Extracts the session identifier from a `Session` header value. +pub(super) fn parse_session_id(session_header: &str) -> Result { + let session_id = session_header.split(';').next().unwrap_or_default().trim(); + if session_id.is_empty() { + return Err(RtspVideoSourceError::InvalidResponse("empty session id")); + } + Ok(session_id.to_owned()) +} + +/// Extracts the `timeout` parameter of a `Session` header value. +pub(super) fn parse_session_timeout_secs(session_header: &str) -> Option { + session_header.split(';').skip(1).find_map(|part| { + let (name, value) = part.trim().split_once('=')?; + if name.trim().eq_ignore_ascii_case("timeout") { + value.trim().parse().ok() + } else { + None + } + }) +} + +/// Extracts the RTP channel from a SETUP response's `Transport` header. +pub(super) fn parse_interleaved_channel( + transport_header: Option<&str>, +) -> Result { + let transport_header = + transport_header.ok_or(RtspVideoSourceError::MissingHeader("Transport"))?; + for part in transport_header.split(';') { + let Some(value) = part.trim().strip_prefix("interleaved=") else { + continue; + }; + if let Some(first) = value.split('-').next().and_then(|channel| channel.parse().ok()) { + return Ok(first); + } + } + Err(RtspVideoSourceError::InvalidResponse("Transport header without interleaved channels")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_credentials_but_strips_them_from_request_uri() { + let url = RtspUrl::parse("rtsp://admin:secret@camera.example:554/live").unwrap(); + + assert_eq!(url.request_uri, "rtsp://camera.example:554/live"); + assert_eq!(url.host_header, "camera.example:554"); + assert_eq!( + url.credentials, + Some(RtspCredentials { username: "admin".to_owned(), password: "secret".to_owned() }) + ); + } + + #[test] + fn percent_decodes_userinfo() { + let url = RtspUrl::parse("rtsp://user%40lk:p%40ss%2Fword@camera.example/live").unwrap(); + assert_eq!( + url.credentials, + Some(RtspCredentials { + username: "user@lk".to_owned(), + password: "p@ss/word".to_owned(), + }) + ); + } + + #[test] + fn defaults_to_port_554() { + let url = RtspUrl::parse("rtsp://camera.example/live").unwrap(); + assert_eq!(url.port, 554); + assert_eq!(url.host_header, "camera.example:554"); + assert_eq!(url.request_uri, "rtsp://camera.example/live"); + } + + #[test] + fn parses_bracketed_ipv6_host() { + let url = RtspUrl::parse("rtsp://[2001:db8::1]:8554/live").unwrap(); + assert_eq!(url.connect_host, "2001:db8::1"); + assert_eq!(url.port, 8554); + assert_eq!(url.host_header, "[2001:db8::1]:8554"); + assert_eq!(url.request_uri, "rtsp://[2001:db8::1]:8554/live"); + } + + #[test] + fn rejects_invalid_urls() { + assert!(matches!( + RtspUrl::parse("http://camera.example/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp:///live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp://:secret@camera.example/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp://camera.example:notaport/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + } + + #[test] + fn parses_response_with_body_and_reports_consumed_bytes() { + let bytes = + b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 4\r\n\r\nbody$leftover"; + let (response, consumed) = parse_response(bytes).unwrap().unwrap(); + + assert_eq!(response.status_code, 200); + assert_eq!(response.reason, "OK"); + assert_eq!(response.header("cseq"), Some("1")); + assert_eq!(response.body, b"body"); + assert_eq!(&bytes[consumed..], b"$leftover"); + } + + #[test] + fn incomplete_response_needs_more_bytes() { + assert!(parse_response(b"RTSP/1.0 200 OK\r\nCSeq:").unwrap().is_none()); + assert!(parse_response(b"RTSP/1.0 200 OK\r\nContent-Length: 4\r\n\r\nbo") + .unwrap() + .is_none()); + } + + #[test] + fn rejects_oversized_header() { + let mut bytes = b"RTSP/1.0 200 OK\r\n".to_vec(); + bytes.resize(MAX_HEADER_BYTES + 8, b'a'); + assert!(matches!( + parse_response(&bytes), + Err(RtspVideoSourceError::InvalidResponse("header too large")) + )); + } + + #[test] + fn parses_session_header() { + assert_eq!(parse_session_id("abc123;timeout=60").unwrap(), "abc123"); + assert!(parse_session_id(" ;timeout=60").is_err()); + assert_eq!(parse_session_timeout_secs("abc123;timeout=60"), Some(60)); + assert_eq!(parse_session_timeout_secs("abc123; Timeout = 30"), Some(30)); + assert_eq!(parse_session_timeout_secs("abc123"), None); + } + + #[test] + fn parses_interleaved_channel() { + assert_eq!( + parse_interleaved_channel(Some("RTP/AVP/TCP;unicast;interleaved=2-3")).unwrap(), + 2 + ); + assert!(matches!( + parse_interleaved_channel(Some("RTP/AVP/TCP;unicast")), + Err(RtspVideoSourceError::InvalidResponse(_)) + )); + assert!(matches!( + parse_interleaved_channel(None), + Err(RtspVideoSourceError::MissingHeader("Transport")) + )); + } +} From ebc1a492c241f42bdaa3dd700a4f7dbde7e2befe Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:52 -0700 Subject: [PATCH 09/38] Implement SDP parsing --- livekit-capture/src/sources/rtsp/sdp.rs | 540 ++++++++++++++++++++++++ 1 file changed, 540 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/sdp.rs diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs new file mode 100644 index 000000000..368820b1e --- /dev/null +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -0,0 +1,540 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! SDP parsing for the video track of an RTSP DESCRIBE response. + +use base64::{engine::general_purpose, Engine as _}; + +use super::{rtp::H26xParameterSets, RtspVideoSourceError}; +use crate::{encoded::EncodedVideoCodec, primitive::VideoResolution}; + +/// RTP timestamp clock rate assumed when the `rtpmap` omits one. +const DEFAULT_CLOCK_RATE: u32 = 90_000; + +/// The video stream selected from an SDP session description. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SdpSession { + /// Selected video track. + pub(super) video: SdpVideoTrack, + /// Aggregate control URL for session-level requests (PLAY, OPTIONS, + /// TEARDOWN), from the session-level `a=control` attribute. + pub(super) aggregate_control_url: String, +} + +/// A video track selected from the SDP. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SdpVideoTrack { + /// RTP payload codec. + pub(super) codec: EncodedVideoCodec, + /// RTP payload type. + pub(super) payload_type: u8, + /// RTP timestamp clock rate. + pub(super) clock_rate: u32, + /// Media control URL used for SETUP. + pub(super) control_url: String, + /// Out-of-band parameter sets from the track's `fmtp` attribute. + pub(super) parameter_sets: H26xParameterSets, + /// Resolution from the track's `a=framesize` attribute, when present. + pub(super) framesize: Option, +} + +#[derive(Debug, Clone, Default)] +struct PartialVideoTrack { + payload_types: Vec, + rtp_maps: Vec, + fmtps: Vec<(u8, String)>, + framesizes: Vec<(u8, VideoResolution)>, + control: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct SdpRtpMap { + payload_type: u8, + codec: EncodedVideoCodec, + clock_rate: u32, +} + +/// Selects a video track from an SDP session description. +/// +/// `base_url` is the URL relative control attributes resolve against: the +/// DESCRIBE response's `Content-Base` when present, else the request URL. +/// With an `expected_codec`, only tracks carrying that codec match; without +/// one, the first supported video track is selected. +pub(super) fn parse_sdp_session( + base_url: &str, + sdp: &str, + expected_codec: Option, +) -> Result { + let mut session_control = None; + let mut tracks = Vec::new(); + let mut current: Option = None; + let mut in_media_section = false; + + for line in sdp.lines().map(str::trim).filter(|line| !line.is_empty()) { + if let Some(media) = line.strip_prefix("m=") { + if let Some(track) = current.take() { + tracks.push(track); + } + in_media_section = true; + if let Some(video) = media.strip_prefix("video ") { + current = Some(parse_video_media(video)); + } + continue; + } + + if !in_media_section { + if let Some(control) = line.strip_prefix("a=control:") { + session_control = Some(control.trim().to_owned()); + } + continue; + } + + let Some(track) = current.as_mut() else { + continue; + }; + if let Some(control) = line.strip_prefix("a=control:") { + track.control = Some(control.trim().to_owned()); + } else if let Some(rtpmap) = line.strip_prefix("a=rtpmap:") { + if let Some(rtp_map) = parse_rtpmap(rtpmap) { + track.rtp_maps.push(rtp_map); + } + } else if let Some(fmtp) = line.strip_prefix("a=fmtp:") { + if let Some((payload_type, params)) = fmtp.trim().split_once(char::is_whitespace) { + if let Ok(payload_type) = payload_type.parse() { + track.fmtps.push((payload_type, params.to_owned())); + } + } + } else if let Some(framesize) = line.strip_prefix("a=framesize:") { + if let Some(parsed) = parse_framesize(framesize) { + track.framesizes.push(parsed); + } + } + } + if let Some(track) = current { + tracks.push(track); + } + + let mut offered = Vec::new(); + for track in tracks { + for payload_type in &track.payload_types { + let Some(rtp_map) = track.rtp_maps.iter().find(|map| map.payload_type == *payload_type) + else { + continue; + }; + if let Some(expected) = expected_codec { + if rtp_map.codec != expected { + if !offered.contains(&rtp_map.codec) { + offered.push(rtp_map.codec); + } + continue; + } + } + + let parameter_sets = track + .fmtps + .iter() + .find(|(fmtp_payload_type, _)| fmtp_payload_type == payload_type) + .map(|(_, params)| parse_fmtp_parameter_sets(rtp_map.codec, params)) + .unwrap_or_default(); + let framesize = track + .framesizes + .iter() + .find(|(framesize_payload_type, _)| framesize_payload_type == payload_type) + .map(|(_, resolution)| *resolution); + + return Ok(SdpSession { + video: SdpVideoTrack { + codec: rtp_map.codec, + payload_type: *payload_type, + clock_rate: rtp_map.clock_rate, + control_url: resolve_control_url(base_url, track.control.as_deref()), + parameter_sets, + framesize, + }, + aggregate_control_url: resolve_control_url(base_url, session_control.as_deref()), + }); + } + } + + match expected_codec { + Some(expected) if !offered.is_empty() => { + Err(RtspVideoSourceError::CodecMismatch { expected, offered }) + } + _ => Err(RtspVideoSourceError::MissingVideoTrack), + } +} + +fn parse_video_media(media: &str) -> PartialVideoTrack { + let payload_types = media + .split_whitespace() + .skip(2) + .filter_map(|payload_type| payload_type.parse().ok()) + .collect(); + PartialVideoTrack { payload_types, ..Default::default() } +} + +fn parse_rtpmap(rtpmap: &str) -> Option { + let (payload_type, encoding) = rtpmap.trim().split_once(' ')?; + let payload_type = payload_type.parse().ok()?; + let mut encoding_parts = encoding.split('/'); + let codec_name = encoding_parts.next()?; + let codec = parse_sdp_codec(codec_name)?; + let clock_rate = encoding_parts + .next() + .and_then(|clock_rate| clock_rate.parse().ok()) + .unwrap_or(DEFAULT_CLOCK_RATE); + Some(SdpRtpMap { payload_type, codec, clock_rate }) +} + +fn parse_sdp_codec(codec_name: &str) -> Option { + if codec_name.eq_ignore_ascii_case("H264") { + Some(EncodedVideoCodec::H264) + } else if codec_name.eq_ignore_ascii_case("H265") || codec_name.eq_ignore_ascii_case("HEVC") { + Some(EncodedVideoCodec::H265) + } else if codec_name.eq_ignore_ascii_case("VP8") { + Some(EncodedVideoCodec::VP8) + } else if codec_name.eq_ignore_ascii_case("VP9") { + Some(EncodedVideoCodec::VP9) + } else if codec_name.eq_ignore_ascii_case("AV1") { + Some(EncodedVideoCodec::AV1) + } else { + None + } +} + +/// Parses an `a=framesize: -` value (RFC 6064). +fn parse_framesize(framesize: &str) -> Option<(u8, VideoResolution)> { + let (payload_type, dimensions) = framesize.trim().split_once(char::is_whitespace)?; + let payload_type = payload_type.parse().ok()?; + let (width, height) = dimensions.trim().split_once('-')?; + let resolution = VideoResolution::new(width.parse().ok()?, height.parse().ok()?); + Some((payload_type, resolution)) +} + +/// Decodes out-of-band parameter sets from an `fmtp` parameter list: +/// `sprop-parameter-sets` for H.264, `sprop-vps`/`sprop-sps`/`sprop-pps` +/// for H.265. Individually malformed entries are skipped. +fn parse_fmtp_parameter_sets(codec: EncodedVideoCodec, params: &str) -> H26xParameterSets { + let mut sets = H26xParameterSets::default(); + for param in params.split(';') { + let Some((name, value)) = param.trim().split_once('=') else { + continue; + }; + match codec { + EncodedVideoCodec::H264 if name.eq_ignore_ascii_case("sprop-parameter-sets") => { + for nal in decode_base64_nals(value) { + // Classify by NAL type rather than position: the + // attribute usually lists SPS then PPS, but not always. + match nal.first().map(|header| header & 0x1f) { + Some(7) => sets.sps.push(nal), + Some(8) => sets.pps.push(nal), + _ => {} + } + } + } + EncodedVideoCodec::H265 => { + let target = if name.eq_ignore_ascii_case("sprop-vps") { + &mut sets.vps + } else if name.eq_ignore_ascii_case("sprop-sps") { + &mut sets.sps + } else if name.eq_ignore_ascii_case("sprop-pps") { + &mut sets.pps + } else { + continue; + }; + target.extend(decode_base64_nals(value)); + } + _ => {} + } + } + sets +} + +fn decode_base64_nals(value: &str) -> Vec> { + value + .split(',') + .map(str::trim) + .filter(|encoded| !encoded.is_empty()) + .filter_map(|encoded| general_purpose::STANDARD.decode(encoded).ok()) + .filter(|nal| !nal.is_empty()) + .collect() +} + +/// Resolves an SDP `control` attribute against the session base URL. +fn resolve_control_url(base_url: &str, control: Option<&str>) -> String { + let Some(control) = control.map(str::trim).filter(|control| !control.is_empty()) else { + return base_url.to_owned(); + }; + if control == "*" { + return base_url.to_owned(); + } + if control.starts_with("rtsp://") { + return control.to_owned(); + } + if control.starts_with('/') { + let authority = base_url + .strip_prefix("rtsp://") + .map(|rest| rest.split('/').next().unwrap_or(rest)) + .unwrap_or_default(); + return format!("rtsp://{authority}{control}"); + } + format!("{}/{}", base_url.trim_end_matches('/'), control) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BASE_URL: &str = "rtsp://camera.example/live"; + + #[test] + fn parses_sdp_video_track() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.payload_type, 96); + assert_eq!(session.video.clock_rate, 90_000); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=1"); + assert_eq!(session.aggregate_control_url, BASE_URL); + assert!(session.video.parameter_sets.is_empty()); + assert_eq!(session.video.framesize, None); + } + + #[test] + fn parses_vp8_vp9_and_av1_sdp_video_tracks() { + for (rtpmap, codec) in [ + ("VP8/90000", EncodedVideoCodec::VP8), + ("VP9/90000", EncodedVideoCodec::VP9), + ("AV1/90000", EncodedVideoCodec::AV1), + ] { + let sdp = format!( + "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 {rtpmap}\r\n" + ); + + let session = parse_sdp_session(BASE_URL, &sdp, Some(codec)).unwrap(); + + assert_eq!(session.video.codec, codec); + assert_eq!(session.video.payload_type, 96); + assert_eq!(session.video.clock_rate, 90_000); + } + } + + #[test] + fn rejects_sdp_codec_mismatch() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 VP9/90000\r\n"; + + let err = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::AV1)).unwrap_err(); + + match err { + RtspVideoSourceError::CodecMismatch { expected, offered } => { + assert_eq!(expected, EncodedVideoCodec::AV1); + assert_eq!(offered, vec![EncodedVideoCodec::VP9]); + } + other => panic!("expected codec mismatch, got {other:?}"), + } + } + + #[test] + fn selects_expected_codec_among_multiple_payload_types() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.payload_type, 96); + } + + #[test] + fn selects_expected_codec_from_later_video_section() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=2\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); + } + + #[test] + fn rejects_sdp_listing_all_offered_codecs_when_none_match() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let err = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::VP8)).unwrap_err(); + + match err { + RtspVideoSourceError::CodecMismatch { expected, offered } => { + assert_eq!(expected, EncodedVideoCodec::VP8); + assert_eq!(offered, vec![EncodedVideoCodec::H265, EncodedVideoCodec::H264]); + } + other => panic!("expected codec mismatch, got {other:?}"), + } + } + + #[test] + fn ignores_audio_sections() { + let sdp = "\ +v=0\r\n\ +m=audio 0 RTP/AVP 97\r\n\ +a=rtpmap:97 PCMU/8000\r\n\ +a=control:trackID=1\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=2\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); + } + + #[test] + fn parses_h264_sprop_parameter_sets() { + // 0x67 (SPS) and 0x68 (PPS) prefixed NAL units. + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=fmtp:96 packetization-mode=1;sprop-parameter-sets=ZwlA,aAlB;profile-level-id=42e01e\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); + assert_eq!(session.video.parameter_sets.pps, vec![vec![0x68, 0x09, 0x41]]); + assert!(session.video.parameter_sets.vps.is_empty()); + } + + #[test] + fn parses_h265_sprop_attributes() { + // 0x40 (VPS), 0x42 (SPS), and 0x44 (PPS) prefixed NAL units. + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H265/90000\r\n\ +a=fmtp:96 sprop-vps=QAEB;sprop-sps=QgEC;sprop-pps=RAED\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.video.parameter_sets.vps, vec![vec![0x40, 0x01, 0x01]]); + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x42, 0x01, 0x02]]); + assert_eq!(session.video.parameter_sets.pps, vec![vec![0x44, 0x01, 0x03]]); + } + + #[test] + fn tolerates_malformed_sprop_entries() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=fmtp:96 sprop-parameter-sets=!!!not-base64!!!,ZwlA\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); + assert!(session.video.parameter_sets.pps.is_empty()); + } + + #[test] + fn parses_framesize_attribute() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=framesize:96 1280-720\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.video.framesize, Some(VideoResolution::new(1280, 720))); + } + + #[test] + fn resolves_session_level_aggregate_control() { + let sdp = "\ +v=0\r\n\ +a=control:rtsp://camera.example/live/aggregate\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.aggregate_control_url, "rtsp://camera.example/live/aggregate"); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=1"); + } + + #[test] + fn star_control_resolves_to_base_url() { + let sdp = "\ +v=0\r\n\ +a=control:*\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + + assert_eq!(session.aggregate_control_url, BASE_URL); + } + + #[test] + fn resolves_absolute_path_control_url() { + assert_eq!( + resolve_control_url(BASE_URL, Some("/stream/trackID=1")), + "rtsp://camera.example/stream/trackID=1" + ); + } + + #[test] + fn resolves_control_against_content_base() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = + parse_sdp_session("rtsp://camera.example/relocated/", sdp, None).unwrap(); + + assert_eq!(session.video.control_url, "rtsp://camera.example/relocated/trackID=1"); + } +} From 9ee43142a9fb9bccc6b2e4f55c100640797e3058 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:52 -0700 Subject: [PATCH 10/38] Implement dimension parsing --- .../src/sources/rtsp/dimensions.rs | 632 ++++++++++++++++++ 1 file changed, 632 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/dimensions.rs diff --git a/livekit-capture/src/sources/rtsp/dimensions.rs b/livekit-capture/src/sources/rtsp/dimensions.rs new file mode 100644 index 000000000..08e044fa6 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/dimensions.rs @@ -0,0 +1,632 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Frame dimension parsing from keyframe payloads, for resolution discovery. +//! +//! Every parser returns `None` on malformed or unexpected input; discovery +//! treats that as "not discoverable" rather than a stream error. + +use super::bits::{read_leb128, BitReader}; +use crate::{ + encoded::{h26x::annex_b_nalus, EncodedVideoCodec}, + primitive::VideoResolution, +}; + +/// Extracts the frame dimensions from a keyframe access-unit payload. +pub(super) fn access_unit_resolution( + codec: EncodedVideoCodec, + payload: &[u8], +) -> Option { + match codec { + EncodedVideoCodec::H264 => annex_b_nalus(payload) + .into_iter() + .find(|nal| nal.first().is_some_and(|header| header & 0x1f == 7)) + .and_then(|nal| sps_resolution(codec, nal)), + EncodedVideoCodec::H265 => annex_b_nalus(payload) + .into_iter() + .find(|nal| nal.first().is_some_and(|header| (header >> 1) & 0x3f == 33)) + .and_then(|nal| sps_resolution(codec, nal)), + EncodedVideoCodec::VP8 => vp8_keyframe_resolution(payload), + EncodedVideoCodec::VP9 => vp9_keyframe_resolution(payload), + EncodedVideoCodec::AV1 => av1_sequence_header_resolution(payload), + } +} + +/// Extracts the frame dimensions from a raw H.264 or H.265 SPS NAL unit, +/// such as one carried out-of-band in SDP `sprop` attributes. +pub(super) fn sps_resolution( + codec: EncodedVideoCodec, + sps_nal: &[u8], +) -> Option { + match codec { + EncodedVideoCodec::H264 => h264_sps_resolution(&rbsp_from_nal(sps_nal, 1)?), + EncodedVideoCodec::H265 => h265_sps_resolution(&rbsp_from_nal(sps_nal, 2)?), + _ => None, + } +} + +/// Strips the NAL header and emulation-prevention bytes (`00 00 03`), which +/// must not reach the bit-level parsers. +fn rbsp_from_nal(nal: &[u8], header_len: usize) -> Option> { + let payload = nal.get(header_len..)?; + let mut rbsp = Vec::with_capacity(payload.len()); + let mut zeros = 0usize; + for &byte in payload { + if zeros >= 2 && byte == 0x03 { + zeros = 0; + continue; + } + zeros = if byte == 0 { zeros + 1 } else { 0 }; + rbsp.push(byte); + } + Some(rbsp) +} + +/// Parses the dimensions from an H.264 SPS RBSP (ITU-T H.264 section 7.3.2.1). +fn h264_sps_resolution(rbsp: &[u8]) -> Option { + let mut reader = BitReader::new(rbsp); + let profile_idc = reader.read_bits(8)?; + reader.skip_bits(8)?; // constraint flags + reserved + reader.skip_bits(8)?; // level_idc + reader.read_ue()?; // seq_parameter_set_id + + let mut chroma_format_idc = 1; + let mut separate_colour_plane = false; + if matches!(profile_idc, 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135) + { + chroma_format_idc = reader.read_ue()?; + if chroma_format_idc == 3 { + separate_colour_plane = reader.read_flag()?; + } + reader.read_ue()?; // bit_depth_luma_minus8 + reader.read_ue()?; // bit_depth_chroma_minus8 + reader.read_bit()?; // qpprime_y_zero_transform_bypass_flag + if reader.read_flag()? { + // seq_scaling_matrix_present_flag + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for index in 0..lists { + if reader.read_flag()? { + skip_h264_scaling_list(&mut reader, if index < 6 { 16 } else { 64 })?; + } + } + } + } + + reader.read_ue()?; // log2_max_frame_num_minus4 + let pic_order_cnt_type = reader.read_ue()?; + if pic_order_cnt_type == 0 { + reader.read_ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } else if pic_order_cnt_type == 1 { + reader.read_bit()?; // delta_pic_order_always_zero_flag + reader.skip_se()?; // offset_for_non_ref_pic + reader.skip_se()?; // offset_for_top_to_bottom_field + let cycle_length = reader.read_ue()?; + for _ in 0..cycle_length { + reader.skip_se()?; // offset_for_ref_frame + } + } + reader.read_ue()?; // max_num_ref_frames + reader.read_bit()?; // gaps_in_frame_num_value_allowed_flag + + let pic_width_in_mbs = reader.read_ue()?.checked_add(1)?; + let pic_height_in_map_units = reader.read_ue()?.checked_add(1)?; + let frame_mbs_only = reader.read_flag()?; + if !frame_mbs_only { + reader.read_bit()?; // mb_adaptive_frame_field_flag + } + reader.read_bit()?; // direct_8x8_inference_flag + + let frame_height_factor = if frame_mbs_only { 1 } else { 2 }; + let mut width = pic_width_in_mbs.checked_mul(16)?; + let mut height = + pic_height_in_map_units.checked_mul(16)?.checked_mul(frame_height_factor)?; + + if reader.read_flag()? { + // frame_cropping_flag + let crop_left = reader.read_ue()?; + let crop_right = reader.read_ue()?; + let crop_top = reader.read_ue()?; + let crop_bottom = reader.read_ue()?; + let chroma_array_type = if separate_colour_plane { 0 } else { chroma_format_idc }; + let (sub_width, sub_height) = match chroma_array_type { + 0 => (1, 1), + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + let crop_unit_x = sub_width; + let crop_unit_y = sub_height * frame_height_factor; + width = width.checked_sub(crop_left.checked_add(crop_right)?.checked_mul(crop_unit_x)?)?; + height = + height.checked_sub(crop_top.checked_add(crop_bottom)?.checked_mul(crop_unit_y)?)?; + } + + checked_resolution(width, height) +} + +fn skip_h264_scaling_list(reader: &mut BitReader<'_>, size: u32) -> Option<()> { + let mut last_scale = 8i32; + let mut next_scale = 8i32; + for _ in 0..size { + if next_scale != 0 { + let delta = read_se(reader)?; + next_scale = (last_scale + delta).rem_euclid(256); + } + if next_scale != 0 { + last_scale = next_scale; + } + } + Some(()) +} + +fn read_se(reader: &mut BitReader<'_>) -> Option { + let code = reader.read_ue()?; + let magnitude = code.div_ceil(2) as i32; + Some(if code % 2 == 1 { magnitude } else { -magnitude }) +} + +/// Parses the dimensions from an H.265 SPS RBSP (ITU-T H.265 section 7.3.2.2). +fn h265_sps_resolution(rbsp: &[u8]) -> Option { + let mut reader = BitReader::new(rbsp); + reader.skip_bits(4)?; // sps_video_parameter_set_id + let max_sub_layers_minus1 = reader.read_bits(3)? as usize; + reader.read_bit()?; // sps_temporal_id_nesting_flag + skip_h265_profile_tier_level(&mut reader, max_sub_layers_minus1)?; + reader.read_ue()?; // sps_seq_parameter_set_id + + let chroma_format_idc = reader.read_ue()?; + if chroma_format_idc == 3 { + reader.read_bit()?; // separate_colour_plane_flag + } + let mut width = reader.read_ue()?; + let mut height = reader.read_ue()?; + + if reader.read_flag()? { + // conformance_window_flag + let left = reader.read_ue()?; + let right = reader.read_ue()?; + let top = reader.read_ue()?; + let bottom = reader.read_ue()?; + let (sub_width, sub_height) = match chroma_format_idc { + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + width = width.checked_sub(left.checked_add(right)?.checked_mul(sub_width)?)?; + height = height.checked_sub(top.checked_add(bottom)?.checked_mul(sub_height)?)?; + } + + checked_resolution(width, height) +} + +/// Skips a `profile_tier_level` structure with `profilePresentFlag = 1`. +fn skip_h265_profile_tier_level( + reader: &mut BitReader<'_>, + max_sub_layers_minus1: usize, +) -> Option<()> { + reader.skip_bits(88)?; // general profile space/tier/idc/compat/constraints + reader.skip_bits(8)?; // general_level_idc + + let mut profile_present = [false; 8]; + let mut level_present = [false; 8]; + for index in 0..max_sub_layers_minus1.min(8) { + profile_present[index] = reader.read_flag()?; + level_present[index] = reader.read_flag()?; + } + if max_sub_layers_minus1 > 0 { + for _ in max_sub_layers_minus1..8 { + reader.skip_bits(2)?; // reserved_zero_2bits + } + } + for index in 0..max_sub_layers_minus1.min(8) { + if profile_present[index] { + reader.skip_bits(88)?; + } + if level_present[index] { + reader.skip_bits(8)?; + } + } + Some(()) +} + +/// Parses the dimensions from a VP8 keyframe payload (RFC 6386 section 9.1). +fn vp8_keyframe_resolution(payload: &[u8]) -> Option { + let header = *payload.first()?; + if header & 0x01 != 0 { + return None; // not a keyframe + } + if payload.get(3..6)? != [0x9d, 0x01, 0x2a] { + return None; // missing keyframe start code + } + let width = u32::from(u16::from_le_bytes([*payload.get(6)?, *payload.get(7)?]) & 0x3fff); + let height = u32::from(u16::from_le_bytes([*payload.get(8)?, *payload.get(9)?]) & 0x3fff); + checked_resolution(width, height) +} + +/// Parses the dimensions from a VP9 keyframe's uncompressed header. +fn vp9_keyframe_resolution(payload: &[u8]) -> Option { + let mut reader = BitReader::new(payload); + if reader.read_bits(2)? != 0b10 { + return None; // frame_marker + } + let profile = reader.read_bit()? | (reader.read_bit()? << 1); + if profile == 3 { + reader.read_bit()?; // reserved_zero + } + if reader.read_flag()? { + return None; // show_existing_frame repeats a decoded frame + } + if reader.read_bit()? != 0 { + return None; // frame_type: not a keyframe + } + reader.read_bit()?; // show_frame + reader.read_bit()?; // error_resilient_mode + if reader.read_bits(24)? != 0x49_83_42 { + return None; // frame_sync_code + } + + // color_config + if profile >= 2 { + reader.read_bit()?; // ten_or_twelve_bit + } + let color_space = reader.read_bits(3)?; + const CS_RGB: u32 = 7; + if color_space != CS_RGB { + reader.read_bit()?; // color_range + if profile == 1 || profile == 3 { + reader.skip_bits(3)?; // subsampling_x, subsampling_y, reserved + } + } else if profile == 1 || profile == 3 { + reader.read_bit()?; // reserved_zero + } + + let width = reader.read_bits(16)?.checked_add(1)?; + let height = reader.read_bits(16)?.checked_add(1)?; + checked_resolution(width, height) +} + +/// Parses the maximum frame dimensions from the sequence header OBU of an +/// AV1 access unit built from size-prefixed OBUs. +fn av1_sequence_header_resolution(payload: &[u8]) -> Option { + let mut cursor = 0; + while cursor < payload.len() { + let header = *payload.get(cursor)?; + if header & 0x80 != 0 { + return None; // obu_forbidden_bit + } + let obu_type = (header & 0x78) >> 3; + let has_extension = header & 0x04 != 0; + let has_size = header & 0x02 != 0; + cursor += if has_extension { 2 } else { 1 }; + if !has_size { + // Without a size field the OBU extends to the end of the unit. + return (obu_type == 1) + .then(|| av1_sequence_header_obu_resolution(payload.get(cursor..)?)) + .flatten(); + } + let size = read_leb128(payload, &mut cursor)?; + let end = cursor.checked_add(size)?; + let obu_payload = payload.get(cursor..end)?; + if obu_type == 1 { + return av1_sequence_header_obu_resolution(obu_payload); + } + cursor = end; + } + None +} + +/// Parses `max_frame_width/height` from a sequence header OBU payload +/// (AV1 specification section 5.5.1). +fn av1_sequence_header_obu_resolution(payload: &[u8]) -> Option { + let mut reader = BitReader::new(payload); + reader.read_bits(3)?; // seq_profile + reader.read_bit()?; // still_picture + let reduced_still_picture_header = reader.read_flag()?; + if reduced_still_picture_header { + reader.skip_bits(5)?; // seq_level_idx[0] + } else { + let mut decoder_model_info_present = false; + let mut buffer_delay_length = 0usize; + if reader.read_flag()? { + // timing_info_present_flag + reader.skip_bits(32)?; // num_units_in_display_tick + reader.skip_bits(32)?; // time_scale + if reader.read_flag()? { + // equal_picture_interval + reader.read_ue()?; // num_ticks_per_picture_minus_1 (uvlc) + } + decoder_model_info_present = reader.read_flag()?; + if decoder_model_info_present { + buffer_delay_length = reader.read_bits(5)? as usize + 1; + reader.skip_bits(32)?; // num_units_in_decoding_tick + reader.skip_bits(10)?; // buffer_removal + frame_presentation lengths + } + } + let initial_display_delay_present = reader.read_flag()?; + let operating_points = reader.read_bits(5)? as usize + 1; + for _ in 0..operating_points { + reader.skip_bits(12)?; // operating_point_idc + let seq_level_idx = reader.read_bits(5)?; + if seq_level_idx > 7 { + reader.read_bit()?; // seq_tier + } + if decoder_model_info_present && reader.read_flag()? { + reader.skip_bits(buffer_delay_length * 2 + 1)?; // operating_parameters_info + } + if initial_display_delay_present && reader.read_flag()? { + reader.skip_bits(4)?; // initial_display_delay_minus_1 + } + } + } + + let width_bits = reader.read_bits(4)? + 1; + let height_bits = reader.read_bits(4)? + 1; + let width = reader.read_bits(width_bits)?.checked_add(1)?; + let height = reader.read_bits(height_bits)?.checked_add(1)?; + checked_resolution(width, height) +} + +fn checked_resolution(width: u32, height: u32) -> Option { + (width > 0 && height > 0).then_some(VideoResolution::new(width, height)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// MSB-first bit writer for composing test bitstreams. + #[derive(Default)] + struct BitWriter { + bytes: Vec, + bit_offset: usize, + } + + impl BitWriter { + fn push_bit(&mut self, bit: u32) { + if self.bit_offset % 8 == 0 { + self.bytes.push(0); + } + let byte = self.bytes.last_mut().unwrap(); + *byte |= ((bit & 1) as u8) << (7 - self.bit_offset % 8); + self.bit_offset += 1; + } + + fn push_bits(&mut self, value: u32, bits: u32) { + for offset in (0..bits).rev() { + self.push_bit((value >> offset) & 1); + } + } + + fn push_ue(&mut self, value: u32) { + let code = value + 1; + let bits = 32 - code.leading_zeros(); + self.push_bits(0, bits - 1); + self.push_bits(code, bits); + } + + fn finish(mut self) -> Vec { + // rbsp_stop_one_bit and alignment. + self.push_bit(1); + while self.bit_offset % 8 != 0 { + self.push_bit(0); + } + self.bytes + } + } + + fn h264_sps_nal(profile_idc: u32, width: u32, height: u32, crop_bottom: u32) -> Vec { + let mut writer = BitWriter::default(); + writer.push_bits(profile_idc, 8); + writer.push_bits(0, 8); // constraint flags + writer.push_bits(30, 8); // level_idc + writer.push_ue(0); // seq_parameter_set_id + if profile_idc == 100 { + writer.push_ue(1); // chroma_format_idc (4:2:0) + writer.push_ue(0); // bit_depth_luma_minus8 + writer.push_ue(0); // bit_depth_chroma_minus8 + writer.push_bit(0); // qpprime_y_zero_transform_bypass_flag + writer.push_bit(0); // seq_scaling_matrix_present_flag + } + writer.push_ue(0); // log2_max_frame_num_minus4 + writer.push_ue(0); // pic_order_cnt_type + writer.push_ue(0); // log2_max_pic_order_cnt_lsb_minus4 + writer.push_ue(1); // max_num_ref_frames + writer.push_bit(0); // gaps_in_frame_num_value_allowed_flag + writer.push_ue(width / 16 - 1); + writer.push_ue(height.div_ceil(16) - 1); + writer.push_bit(1); // frame_mbs_only_flag + writer.push_bit(1); // direct_8x8_inference_flag + if crop_bottom > 0 { + writer.push_bit(1); // frame_cropping_flag + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(crop_bottom / 2); // CropUnitY = 2 for 4:2:0 + } else { + writer.push_bit(0); + } + writer.push_bit(0); // vui_parameters_present_flag + + let mut nal = vec![0x67]; + nal.extend(writer.finish()); + nal + } + + fn h265_sps_nal(width: u32, height: u32, crop_bottom: u32) -> Vec { + let mut writer = BitWriter::default(); + writer.push_bits(0, 4); // sps_video_parameter_set_id + writer.push_bits(0, 3); // sps_max_sub_layers_minus1 + writer.push_bit(1); // sps_temporal_id_nesting_flag + writer.push_bits(0, 32); // profile_tier_level: space/tier/idc/compat... + writer.push_bits(0, 32); + writer.push_bits(0, 24); // ...constraint and reserved bits (88 total) + writer.push_bits(93, 8); // general_level_idc + writer.push_ue(0); // sps_seq_parameter_set_id + writer.push_ue(1); // chroma_format_idc (4:2:0) + writer.push_ue(width); + writer.push_ue(height + crop_bottom * 2); + if crop_bottom > 0 { + writer.push_bit(1); // conformance_window_flag + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(crop_bottom); // SubHeightC = 2 for 4:2:0 + } else { + writer.push_bit(0); + } + + let mut nal = vec![0x42, 0x01]; + nal.extend(writer.finish()); + nal + } + + #[test] + fn parses_h264_sps_dimensions() { + let nal = h264_sps_nal(66, 640, 480, 0); + assert_eq!( + sps_resolution(EncodedVideoCodec::H264, &nal), + Some(VideoResolution::new(640, 480)) + ); + } + + #[test] + fn parses_h264_high_profile_sps_with_cropping() { + // 1920x1088 coded, cropped to 1920x1080. + let nal = h264_sps_nal(100, 1920, 1080, 8); + assert_eq!( + sps_resolution(EncodedVideoCodec::H264, &nal), + Some(VideoResolution::new(1920, 1080)) + ); + } + + #[test] + fn parses_h264_sps_from_access_unit() { + let sps = h264_sps_nal(66, 1280, 720, 0); + let mut payload = Vec::new(); + payload.extend_from_slice(&[0, 0, 0, 1]); + payload.extend_from_slice(&sps); + payload.extend_from_slice(&[0, 0, 0, 1, 0x68, 0x08]); + payload.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88, 0x84]); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::H264, &payload), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_h265_sps_dimensions() { + let nal = h265_sps_nal(1280, 720, 0); + assert_eq!( + sps_resolution(EncodedVideoCodec::H265, &nal), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_h265_sps_with_conformance_window() { + // 1920x1088 coded, cropped to 1920x1080. + let nal = h265_sps_nal(1920, 1080, 4); + assert_eq!( + sps_resolution(EncodedVideoCodec::H265, &nal), + Some(VideoResolution::new(1920, 1080)) + ); + } + + #[test] + fn strips_emulation_prevention_bytes() { + assert_eq!(rbsp_from_nal(&[0x67, 0x01, 0x00, 0x00, 0x03, 0x02], 1), Some(vec![ + 0x01, 0x00, 0x00, 0x02 + ])); + // The escape only applies after two zero bytes. + assert_eq!(rbsp_from_nal(&[0x67, 0x01, 0x00, 0x03, 0x02], 1), Some(vec![ + 0x01, 0x00, 0x03, 0x02 + ])); + } + + #[test] + fn parses_vp8_keyframe_dimensions() { + let payload = [ + 0x00, 0x00, 0x00, // frame tag: keyframe + 0x9d, 0x01, 0x2a, // start code + 0x80, 0x02, // width 640 + 0xe0, 0x01, // height 480 + ]; + assert_eq!( + access_unit_resolution(EncodedVideoCodec::VP8, &payload), + Some(VideoResolution::new(640, 480)) + ); + } + + #[test] + fn rejects_vp8_delta_frame() { + let payload = [0x01, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01]; + assert_eq!(access_unit_resolution(EncodedVideoCodec::VP8, &payload), None); + } + + #[test] + fn parses_vp9_keyframe_dimensions() { + let mut writer = BitWriter::default(); + writer.push_bits(0b10, 2); // frame_marker + writer.push_bits(0, 2); // profile 0 + writer.push_bit(0); // show_existing_frame + writer.push_bit(0); // frame_type: keyframe + writer.push_bit(1); // show_frame + writer.push_bit(0); // error_resilient_mode + writer.push_bits(0x49_83_42, 24); // frame_sync_code + writer.push_bits(0, 3); // color_space + writer.push_bit(0); // color_range + writer.push_bits(1280 - 1, 16); + writer.push_bits(720 - 1, 16); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::VP9, &writer.finish()), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_av1_sequence_header_dimensions() { + let mut writer = BitWriter::default(); + writer.push_bits(0, 3); // seq_profile + writer.push_bit(0); // still_picture + writer.push_bit(0); // reduced_still_picture_header + writer.push_bit(0); // timing_info_present_flag + writer.push_bit(0); // initial_display_delay_present_flag + writer.push_bits(0, 5); // operating_points_cnt_minus_1 + writer.push_bits(0, 12); // operating_point_idc[0] + writer.push_bits(5, 5); // seq_level_idx[0] + writer.push_bits(10, 4); // frame_width_bits_minus_1 + writer.push_bits(9, 4); // frame_height_bits_minus_1 + writer.push_bits(1280 - 1, 11); // max_frame_width_minus_1 + writer.push_bits(720 - 1, 10); // max_frame_height_minus_1 + let obu_payload = writer.finish(); + + // Size-prefixed sequence header OBU followed by an unrelated OBU. + let mut payload = vec![0x0a, obu_payload.len() as u8]; + payload.extend(&obu_payload); + payload.extend_from_slice(&[0x32, 0x01, 0x10]); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::AV1, &payload), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn skips_leading_av1_obus_without_sequence_header() { + let payload = [0x12, 0x00, 0x32, 0x01, 0x10]; // temporal delimiter + frame + assert_eq!(access_unit_resolution(EncodedVideoCodec::AV1, &payload), None); + } +} From d21cedbdc3ebf2587da8930f1d1fea1ede756351 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:53 -0700 Subject: [PATCH 11/38] Implement source --- livekit-capture/src/sources/mod.rs | 3 + livekit-capture/src/sources/rtsp/mod.rs | 1082 +++++++++++++++++++++++ 2 files changed, 1085 insertions(+) create mode 100644 livekit-capture/src/sources/rtsp/mod.rs diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 732fb8bdd..377a0ff73 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -23,3 +23,6 @@ pub mod gstreamer; #[cfg(feature = "source-pattern")] pub mod pattern; + +#[cfg(feature = "source-rtsp")] +pub mod rtsp; diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs new file mode 100644 index 000000000..6ac55478c --- /dev/null +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -0,0 +1,1082 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Encoded video capture from an RTSP server. +//! +//! [`RtspVideoSource`] connects to an `rtsp://` URL, negotiates a video +//! stream over TCP-interleaved RTP, and yields its access units without +//! re-encoding. Basic and Digest authentication are supported, and packet +//! loss is recovered by waiting for the next keyframe. +//! +//! The connection is not re-established on failure: a connection error ends +//! the source with an error, and a clean server-side end of stream ends it +//! like any finite source. + +mod auth; +mod bits; +mod client; +mod dimensions; +mod rtp; +mod sdp; + +use std::{ + fmt, io, str, + time::{Duration, Instant}, +}; + +use thiserror::Error; + +use crate::{ + encoded::{EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, OwnedEncodedAccessUnit}, + error::SourceError, + primitive::VideoResolution, + pump::PumpStop, +}; +use auth::RtspCredentials; +use client::{ + parse_interleaved_channel, parse_session_id, parse_session_timeout_secs, InterleavedPoll, + RtspClient, RtspUrl, +}; +use rtp::RtpAccessUnitAssembler; + +/// Default TCP connect and handshake timeout. +const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Default maximum stream silence tolerated before the source fails. +const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(30); + +/// How long resolution discovery waits for the stream's first keyframe. +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(10); + +/// Session timeout assumed when the server's `Session` header declares none +/// (RFC 2326 section 12.37). +const DEFAULT_SESSION_TIMEOUT_SECS: u64 = 60; + +/// Configuration for an RTSP encoded video source. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct RtspVideoSourceConfig { + /// RTSP URL (`rtsp://host[:port]/path`). + /// + /// URL userinfo (`rtsp://user:password@...`) is accepted and stripped + /// from requests; [`Self::username`] and [`Self::password`] take + /// precedence over it. + pub url: String, + + /// Username for RTSP authentication, overriding URL userinfo. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub username: Option, + + /// Password for RTSP authentication, overriding URL userinfo. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub password: Option, + + /// Codec required from the stream. When omitted, the first supported + /// video track offered by the SDP is used. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub codec: Option, + + /// Encoded frame resolution. + /// + /// When omitted, the resolution is discovered from the SDP when it + /// declares one, and from the stream's first keyframe otherwise, so + /// construction may wait for the stream to produce data. When set, + /// construction returns without waiting, and the first keyframe is + /// verified against this value. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub resolution: Option, + + /// TCP connect and RTSP handshake timeout in milliseconds + /// (default 10000). + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub connect_timeout_ms: Option, + + /// Maximum stream silence tolerated before the source fails, in + /// milliseconds (default 30000). Receiving any stream bytes resets the + /// limit. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub idle_timeout_ms: Option, +} + +/// Protocol phase a timeout occurred in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RtspPhase { + /// Establishing the TCP connection. + Connect, + /// Waiting for the DESCRIBE response. + Describe, + /// Waiting for the SETUP response. + Setup, + /// Waiting for the PLAY response. + Play, + /// Waiting for interleaved stream data. + Stream, +} + +impl fmt::Display for RtspPhase { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Connect => "the TCP connection", + Self::Describe => "the DESCRIBE response", + Self::Setup => "the SETUP response", + Self::Play => "the PLAY response", + Self::Stream => "stream data", + }) + } +} + +/// Error returned by RTSP encoded video sources. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum RtspVideoSourceError { + /// I/O on the RTSP connection failed. + #[error("RTSP I/O failed: {0}")] + Io(#[from] io::Error), + /// A protocol phase exceeded its timeout. + #[error("RTSP timed out waiting for {phase}")] + Timeout { + /// Protocol phase the client was waiting on. + phase: RtspPhase, + }, + /// The RTSP URL was invalid or unsupported. + #[error("invalid RTSP URL: {0}")] + InvalidUrl(&'static str), + /// The server returned a non-success status. + #[error("RTSP request failed with status {code} {reason}")] + RtspStatus { + /// RTSP status code. + code: u16, + /// RTSP status reason. + reason: String, + }, + /// A response was malformed. + #[error("invalid RTSP response: {0}")] + InvalidResponse(&'static str), + /// A response was missing a required header. + #[error("RTSP response missing {0} header")] + MissingHeader(&'static str), + /// The server requires authentication but no credentials were supplied. + #[error("RTSP authentication required but no credentials were supplied")] + MissingCredentials, + /// The authentication challenge was malformed. + #[error("invalid RTSP authentication challenge")] + InvalidAuthChallenge, + /// The authentication scheme is not supported. + #[error("unsupported RTSP authentication scheme: {0}")] + UnsupportedAuthScheme(String), + /// The Digest algorithm is not supported. + #[error("unsupported RTSP Digest algorithm: {0}")] + UnsupportedDigestAlgorithm(String), + /// The SDP was missing a supported video track. + #[error("RTSP SDP does not contain a supported video track")] + MissingVideoTrack, + /// The SDP did not offer the requested codec on any video track. + #[error("RTSP SDP codec mismatch: expected {expected:?}, offered {offered:?}")] + CodecMismatch { + /// Codec required by the configuration. + expected: EncodedVideoCodec, + /// Supported codecs the SDP video tracks offered instead. + offered: Vec, + }, + /// The SDP body was malformed or not valid UTF-8. + #[error("invalid RTSP SDP")] + InvalidSdp, + /// Interleaved framing was malformed or a non-interleaved byte arrived. + #[error("unexpected RTSP interleaved data")] + UnexpectedData, + /// The stream produced no keyframe during resolution discovery. + #[error( + "stream produced no keyframe during resolution discovery; declare `resolution` in the \ + configuration to skip discovery" + )] + DiscoveryTimeout, + /// The first keyframe did not carry parseable dimensions. + #[error( + "could not determine the stream resolution from its first keyframe; declare \ + `resolution` in the configuration" + )] + DiscoveryFailed, + /// The stream ended before resolution discovery completed. + #[error("stream ended before resolution discovery completed")] + EndedDuringDiscovery, + /// The stream's resolution does not match the established one. + #[error("stream produces {actual}, but the source was configured for {configured}")] + ResolutionMismatch { + /// Resolution the source was configured for or discovered. + configured: VideoResolution, + /// Resolution the stream's keyframe declares. + actual: VideoResolution, + }, + /// RTP depacketization failed. + #[error("invalid RTP stream: {0}")] + Rtp(Box), +} + +impl From for RtspVideoSourceError { + fn from(err: rtp::RtpDepacketizerError) -> Self { + Self::Rtp(Box::new(err)) + } +} + +/// Progress from one bounded poll of the stream. +enum Poll { + AccessUnit(OwnedEncodedAccessUnit), + TimedOut, + EndOfStream, +} + +/// Encoded source that plays a video stream from an RTSP server. +pub struct RtspVideoSource { + client: RtspClient, + assembler: RtpAccessUnitAssembler, + session_id: String, + aggregate_control_url: String, + rtp_channel: u8, + keepalive_interval: Duration, + keepalive_due: Instant, + idle_timeout: Duration, + codec: EncodedVideoCodec, + resolution: VideoResolution, + resolution_verified: bool, + // Access unit pulled during resolution discovery, handed out first. + pending: Option, + eof: bool, +} + +impl fmt::Debug for RtspVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspVideoSource") + .field("codec", &self.codec) + .field("resolution", &self.resolution) + .field("session_id", &self.session_id) + .finish_non_exhaustive() + } +} + +impl RtspVideoSource { + /// Creates the source. Connection, handshake, and resolution discovery + /// run on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`RtspVideoSource::new_blocking`] outside of async contexts. + #[cfg(feature = "tokio")] + pub async fn new(config: RtspVideoSourceConfig) -> Result { + crate::utils::run_blocking(move || Self::new_blocking(config)).await + } + + /// Connects to the RTSP server and starts playback. + /// + /// The handshake (DESCRIBE, SETUP, PLAY over TCP-interleaved transport) + /// is bounded by the connect timeout. Construction fails on an invalid + /// URL, a missing or codec-mismatched video track, an authentication + /// failure, or any non-success response. + /// + /// When the configuration declares no resolution and the SDP none + /// either, this blocks until the stream's first keyframe arrives + /// (bounded by a timeout) to read the dimensions from it. + pub fn new_blocking(config: RtspVideoSourceConfig) -> Result { + Self::connect(config).map_err(SourceError::new) + } + + fn connect(config: RtspVideoSourceConfig) -> Result { + let url = RtspUrl::parse(&config.url)?; + let credentials = merge_credentials(&config, &url); + let connect_timeout = duration_ms(config.connect_timeout_ms, DEFAULT_CONNECT_TIMEOUT); + let idle_timeout = duration_ms(config.idle_timeout_ms, DEFAULT_IDLE_TIMEOUT); + let handshake_deadline = Instant::now() + connect_timeout; + + let mut client = RtspClient::connect(&url, credentials, handshake_deadline)?; + + let describe = client.request( + "DESCRIBE", + &url.request_uri, + &[("Accept", "application/sdp")], + handshake_deadline, + RtspPhase::Describe, + )?; + let sdp_text = + str::from_utf8(&describe.body).map_err(|_| RtspVideoSourceError::InvalidSdp)?; + // Relative control URLs resolve against the Content-Base per + // RFC 2326 appendix C.1.1, falling back to the request URL. + let base_url = describe + .header("content-base") + .or_else(|| describe.header("content-location")) + .unwrap_or(&url.request_uri); + let session = sdp::parse_sdp_session(base_url, sdp_text, config.codec)?; + + let setup = client.request( + "SETUP", + &session.video.control_url, + &[("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1")], + handshake_deadline, + RtspPhase::Setup, + )?; + let session_header = + setup.header("session").ok_or(RtspVideoSourceError::MissingHeader("Session"))?; + let session_id = parse_session_id(session_header)?; + let session_timeout_secs = + parse_session_timeout_secs(session_header).unwrap_or(DEFAULT_SESSION_TIMEOUT_SECS); + let rtp_channel = parse_interleaved_channel(setup.header("transport"))?; + + client.request( + "PLAY", + &session.aggregate_control_url, + &[("Session", session_id.as_str()), ("Range", "npt=0.000-")], + handshake_deadline, + RtspPhase::Play, + )?; + + // Resolution declared in the configuration wins; otherwise use the + // SDP's hints (`a=framesize`, an out-of-band SPS), which spare + // waiting for media; otherwise discover from the first keyframe. + let codec = session.video.codec; + let sdp_resolution = config.resolution.or(session.video.framesize).or_else(|| { + session + .video + .parameter_sets + .sps + .iter() + .find_map(|sps| dimensions::sps_resolution(codec, sps)) + }); + + let assembler = RtpAccessUnitAssembler::new( + codec, + session.video.payload_type, + session.video.clock_rate, + session.video.parameter_sets.clone(), + sdp_resolution.unwrap_or_default(), + )?; + + let keepalive_interval = Duration::from_secs((session_timeout_secs / 2).max(1)); + let mut source = Self { + client, + assembler, + session_id, + aggregate_control_url: session.aggregate_control_url, + rtp_channel, + keepalive_interval, + keepalive_due: Instant::now() + keepalive_interval, + idle_timeout, + codec, + resolution: sdp_resolution.unwrap_or_default(), + resolution_verified: false, + pending: None, + eof: false, + }; + + // From here on the session is live, so any failure path runs the + // source's TEARDOWN through `Drop`. + if sdp_resolution.is_none() { + source.discover_resolution()?; + } + + log::info!( + "RTSP stream ready: {:?} {} ({} resolution)", + source.codec, + source.resolution, + if config.resolution.is_some() { "declared" } else { "discovered" }, + ); + Ok(source) + } + + /// Blocks until the stream's first keyframe reveals the resolution; the + /// keyframe is kept so it is not lost to discovery. + fn discover_resolution(&mut self) -> Result<(), RtspVideoSourceError> { + let deadline = Instant::now() + DISCOVERY_TIMEOUT; + loop { + match self.poll_access_unit()? { + Poll::AccessUnit(mut access_unit) => { + if access_unit.frame_type != EncodedFrameType::Key { + // Undecodable without the keyframe; the pump would + // drop these pre-roll deltas anyway. + continue; + } + let resolution = + dimensions::access_unit_resolution(self.codec, &access_unit.payload) + .ok_or(RtspVideoSourceError::DiscoveryFailed)?; + access_unit.resolution = resolution; + self.resolution = resolution; + self.assembler.set_resolution(resolution); + // The dimensions came from this keyframe itself. + self.resolution_verified = true; + self.pending = Some(access_unit); + return Ok(()); + } + Poll::TimedOut => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::DiscoveryTimeout); + } + } + Poll::EndOfStream => return Err(RtspVideoSourceError::EndedDuringDiscovery), + } + } + } + + /// Advances the stream by at most one bounded read, running the + /// keepalive and idle-limit bookkeeping. + fn poll_access_unit(&mut self) -> Result { + loop { + if let Some(access_unit) = self.assembler.pop_ready() { + return Ok(Poll::AccessUnit(access_unit)); + } + self.maybe_send_keepalive()?; + + match self.client.poll_unit()? { + InterleavedPoll::Frame { channel, payload } if channel == self.rtp_channel => { + self.assembler.push(&payload)?; + } + // Other channels carry RTCP and non-selected media. + InterleavedPoll::Frame { .. } => {} + InterleavedPoll::Response(response) => { + // In-band responses answer keepalives; a failure (for + // example 454 Session Not Found) means the session died. + if !response.is_success() { + return Err(RtspVideoSourceError::RtspStatus { + code: response.status_code, + reason: response.reason, + }); + } + } + InterleavedPoll::TimedOut => { + if self.client.idle_for() >= self.idle_timeout { + return Err(RtspVideoSourceError::Timeout { phase: RtspPhase::Stream }); + } + return Ok(Poll::TimedOut); + } + InterleavedPoll::EndOfStream => return Ok(Poll::EndOfStream), + } + } + } + + /// Sends an OPTIONS keepalive when one is due, so the server keeps the + /// session alive across stream silence. The reply arrives in-band. + fn maybe_send_keepalive(&mut self) -> Result<(), RtspVideoSourceError> { + if Instant::now() < self.keepalive_due { + return Ok(()); + } + self.client.write_request( + "OPTIONS", + &self.aggregate_control_url, + &[("Session", self.session_id.as_str())], + )?; + self.keepalive_due = Instant::now() + self.keepalive_interval; + Ok(()) + } + + /// Verifies the established resolution against the first keyframe that + /// carries parseable dimensions. + fn verify_resolution( + &mut self, + access_unit: &OwnedEncodedAccessUnit, + ) -> Result<(), RtspVideoSourceError> { + if self.resolution_verified || access_unit.frame_type != EncodedFrameType::Key { + return Ok(()); + } + self.resolution_verified = true; + let Some(actual) = dimensions::access_unit_resolution(self.codec, &access_unit.payload) + else { + log::debug!("RTSP keyframe carries no parseable dimensions; skipping verification"); + return Ok(()); + }; + if actual != self.resolution { + return Err(RtspVideoSourceError::ResolutionMismatch { + configured: self.resolution, + actual, + }); + } + Ok(()) + } +} + +impl Drop for RtspVideoSource { + fn drop(&mut self) { + // Best-effort TEARDOWN so servers with session limits release this + // session immediately instead of waiting out its timeout. The reply + // is intentionally not awaited. + let uri = self.aggregate_control_url.clone(); + let session_id = self.session_id.clone(); + let _ = self.client.write_request("TEARDOWN", &uri, &[("Session", session_id.as_str())]); + } +} + +impl EncodedVideoSource for RtspVideoSource { + fn resolution(&self) -> VideoResolution { + self.resolution + } + + fn codec(&self) -> EncodedVideoCodec { + self.codec + } + + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + if self.eof { + return Ok(None); + } + if let Some(pending) = self.pending.take() { + return Ok(Some(pending)); + } + + // Every read is bounded by the client's socket read timeout, so the + // stop token, the keepalive timer, and the idle limit are all + // observed within roughly 100ms even while the stream is silent. + loop { + if stop.is_stopped() { + return Ok(None); + } + match self.poll_access_unit().map_err(SourceError::new)? { + Poll::AccessUnit(access_unit) => { + self.verify_resolution(&access_unit).map_err(SourceError::new)?; + return Ok(Some(access_unit)); + } + Poll::TimedOut => {} + Poll::EndOfStream => { + self.eof = true; + return Ok(None); + } + } + } + } + + // TODO: Implement `request_keyframe` by sending a best-effort RTCP PLI + // on the interleaved RTCP channel (`rtp_channel + 1`), so late + // subscribers get a keyframe immediately instead of waiting out the + // producer's keyframe interval. +} + +/// Applies the config's per-field credential overrides to the URL userinfo. +fn merge_credentials( + config: &RtspVideoSourceConfig, + url: &RtspUrl, +) -> Option { + let (url_username, url_password) = match &url.credentials { + Some(credentials) => { + (Some(credentials.username.clone()), Some(credentials.password.clone())) + } + None => (None, None), + }; + let username = config.username.clone().or(url_username)?; + let password = config.password.clone().or(url_password).unwrap_or_default(); + Some(RtspCredentials { username, password }) +} + +fn duration_ms(ms: Option, default: Duration) -> Duration { + ms.map(|ms| Duration::from_millis(ms.into())).unwrap_or(default) +} + +#[cfg(test)] +mod tests { + use std::{ + io::{Read, Write}, + net::{TcpListener, TcpStream}, + thread, + }; + + use super::*; + + const SDP_H264: &str = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + const SDP_VP8: &str = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 VP8/90000\r\n"; + + fn config(url: String) -> RtspVideoSourceConfig { + RtspVideoSourceConfig { + url, + username: None, + password: None, + codec: None, + resolution: Some(VideoResolution::new(640, 480)), + connect_timeout_ms: Some(2_000), + idle_timeout_ms: None, + } + } + + fn rtp_packet(sequence_number: u16, timestamp: u32, marker: bool, payload: &[u8]) -> Vec { + let mut packet = Vec::with_capacity(12 + payload.len()); + packet.push(0x80); + packet.push(if marker { 0x80 | 96 } else { 96 }); + packet.extend_from_slice(&sequence_number.to_be_bytes()); + packet.extend_from_slice(×tamp.to_be_bytes()); + packet.extend_from_slice(&0x1122_3344_u32.to_be_bytes()); + packet.extend_from_slice(payload); + packet + } + + fn interleaved(channel: u8, payload: &[u8]) -> Vec { + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.push(b'$'); + frame.push(channel); + frame.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + frame.extend_from_slice(payload); + frame + } + + fn read_request(stream: &mut impl Read) -> String { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + String::from_utf8(request).unwrap() + } + + fn write_response(stream: &mut impl Write, cseq: u32, headers: &[(&str, &str)], body: &[u8]) { + write_status_response(stream, cseq, headers, body, 200, "OK"); + } + + fn write_status_response( + stream: &mut impl Write, + cseq: u32, + headers: &[(&str, &str)], + body: &[u8], + status_code: u16, + reason: &str, + ) { + write!(stream, "RTSP/1.0 {status_code} {reason}\r\nCSeq: {cseq}\r\n").unwrap(); + for (name, value) in headers { + write!(stream, "{name}: {value}\r\n").unwrap(); + } + write!(stream, "\r\n").unwrap(); + if !body.is_empty() { + stream.write_all(body).unwrap(); + } + stream.flush().unwrap(); + } + + /// Answers DESCRIBE/SETUP/PLAY on `stream` and returns once the session + /// is playing on interleaved channels 0-1. + fn serve_handshake(stream: &mut TcpStream, sdp: &str) { + let describe = read_request(stream); + assert!(describe.starts_with("DESCRIBE rtsp://")); + write_response( + stream, + 1, + &[("Content-Type", "application/sdp"), ("Content-Length", &sdp.len().to_string())], + sdp.as_bytes(), + ); + + let setup = read_request(stream); + assert!(setup.starts_with("SETUP rtsp://")); + assert!(setup.contains("Transport: RTP/AVP/TCP;unicast;interleaved=0-1")); + write_response( + stream, + 2, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1"), + ], + &[], + ); + + let play = read_request(stream); + assert!(play.starts_with("PLAY rtsp://")); + assert!(play.contains("Session: abc123")); + write_response(stream, 3, &[], &[]); + } + + #[test] + fn connects_and_reads_rtsp_access_unit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let describe = read_request(&mut stream); + assert!(describe.starts_with("DESCRIBE rtsp://")); + write_response( + &mut stream, + 1, + &[ + ("Content-Type", "application/sdp"), + ("Content-Length", &SDP_H264.len().to_string()), + ], + SDP_H264.as_bytes(), + ); + + let setup = read_request(&mut stream); + assert!(setup.starts_with("SETUP rtsp://")); + write_response( + &mut stream, + 2, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=2-3"), + ], + &[], + ); + + let play = read_request(&mut stream); + assert!(play.starts_with("PLAY rtsp://")); + assert!(play.contains("Session: abc123")); + assert!(play.contains("Range: npt=0.000-")); + write_response(&mut stream, 3, &[], &[]); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(2, &packet)).unwrap(); + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.codec(), EncodedVideoCodec::H264); + assert_eq!(source.resolution(), VideoResolution::new(640, 480)); + assert_eq!(source.session_id, "abc123"); + assert_eq!(source.rtp_channel, 2); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn connects_with_rtsp_digest_auth() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let first_describe = read_request(&mut stream); + assert!(first_describe.starts_with(&format!("DESCRIBE rtsp://{addr}/camera"))); + assert!(!first_describe.contains("Authorization:")); + write_status_response( + &mut stream, + 1, + &[("WWW-Authenticate", "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"")], + &[], + 401, + "Unauthorized", + ); + + let second_describe = read_request(&mut stream); + assert!(!second_describe.contains("admin:secret@")); + assert!(second_describe.contains("Authorization: Digest username=\"admin\"")); + assert!(second_describe.contains(&format!("uri=\"rtsp://{addr}/camera\""))); + assert!(second_describe.contains("qop=auth")); + write_response( + &mut stream, + 2, + &[ + ("Content-Type", "application/sdp"), + ("Content-Length", &SDP_H264.len().to_string()), + ], + SDP_H264.as_bytes(), + ); + + let setup = read_request(&mut stream); + assert!(setup.contains("Authorization: Digest username=\"admin\"")); + write_response( + &mut stream, + 3, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1"), + ], + &[], + ); + + let play = read_request(&mut stream); + assert!(play.contains("Authorization: Digest username=\"admin\"")); + write_response(&mut stream, 4, &[], &[]); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(0, &packet)).unwrap(); + }); + + let mut source = RtspVideoSource::new_blocking(config(format!( + "rtsp://admin:secret@{addr}/camera" + ))) + .unwrap(); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn discovers_resolution_from_first_keyframe() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_VP8); + + // VP8 keyframe with a 10-byte uncompressed header for 640x480. + let vp8_keyframe = [ + 0x10, // payload descriptor: start of partition 0 + 0x00, 0x00, 0x00, // frame tag: keyframe + 0x9d, 0x01, 0x2a, // start code + 0x80, 0x02, // width 640 + 0xe0, 0x01, // height 480 + ]; + let packet = rtp_packet(10, 12_000, true, &vp8_keyframe); + stream.write_all(&interleaved(0, &packet)).unwrap(); + read_request(&mut stream) // TEARDOWN + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: None, + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.resolution(), VideoResolution::new(640, 480)); + + // The discovery keyframe is handed out first, stamped with the + // discovered resolution. + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.resolution, VideoResolution::new(640, 480)); + drop(source); + + let teardown = server.join().unwrap(); + assert!(teardown.starts_with("TEARDOWN rtsp://")); + assert!(teardown.contains("Session: abc123")); + } + + #[test] + fn uses_sdp_framesize_without_waiting_for_media() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=framesize:96 1280-720\r\n"; + serve_handshake(&mut stream, sdp); + // Send no media: construction must not wait for any. + read_request(&mut stream) // TEARDOWN + }); + + let source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: None, + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.resolution(), VideoResolution::new(1280, 720)); + drop(source); + server.join().unwrap(); + } + + #[test] + fn sends_keepalive_during_stream_silence() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + + // Send no interleaved data; the keepalive must arrive during the + // silence. Only then reply and send the first video frame. + let keepalive = read_request(&mut stream); + write_response(&mut stream, 4, &[], &[]); + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(0, &packet)).unwrap(); + keepalive + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + source.keepalive_due = Instant::now() + Duration::from_millis(250); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + + let keepalive = server.join().unwrap(); + assert!(keepalive.starts_with("OPTIONS rtsp://")); + assert!(keepalive.contains("Session: abc123")); + } + + #[test] + fn failed_keepalive_reply_surfaces_as_error() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // The session died server-side: answer in-band with 454. + write_status_response(&mut stream, 4, &[], &[], 454, "Session Not Found"); + thread::sleep(Duration::from_millis(200)); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("454")); + drop(source); + server.join().unwrap(); + } + + #[test] + fn recovers_interleaved_framing_across_read_timeouts() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + let frame = interleaved(0, &packet); + // Split inside the 4-byte interleaved header and pause long + // enough for several client read timeouts in between. + let (head, tail) = frame.split_at(2); + stream.write_all(head).unwrap(); + stream.flush().unwrap(); + thread::sleep(Duration::from_millis(350)); + stream.write_all(tail).unwrap(); + stream.flush().unwrap(); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn stream_silence_times_out_after_idle_limit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // Stay silent past the client's idle limit. + thread::sleep(Duration::from_millis(700)); + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + idle_timeout_ms: Some(300), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + + let stop = PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("stream data"), "unexpected error: {err}"); + drop(source); + server.join().unwrap(); + } + + #[test] + fn stop_token_is_observed_during_stream_silence() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // Stay silent; the client must return on its stop token alone. + thread::sleep(Duration::from_millis(700)); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let stop_signal = stop.clone(); + let stopper = thread::spawn(move || { + thread::sleep(Duration::from_millis(150)); + stop_signal.stop(); + }); + + let started = Instant::now(); + let result = source.next_access_unit(&stop).unwrap(); + assert!(result.is_none()); + assert!( + started.elapsed() < Duration::from_millis(600), + "stop took {:?}", + started.elapsed() + ); + stopper.join().unwrap(); + drop(source); + server.join().unwrap(); + } + + #[test] + fn handshake_read_timeout_is_hard_error() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _describe = read_request(&mut stream); + // Never respond; hold the connection open past the deadline. + thread::sleep(Duration::from_millis(600)); + }); + + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + connect_timeout_ms: Some(300), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap_err(); + + assert!( + err.to_string().contains("DESCRIBE"), + "expected DESCRIBE timeout, got: {err}" + ); + server.join().unwrap(); + } + + #[test] + fn explicit_credentials_override_url_userinfo() { + let url = RtspUrl::parse("rtsp://old:stale@camera.example/live").unwrap(); + let overriding = RtspVideoSourceConfig { + username: Some("new".to_owned()), + password: None, + ..config("rtsp://camera.example/live".to_owned()) + }; + + let credentials = merge_credentials(&overriding, &url).unwrap(); + assert_eq!(credentials.username, "new"); + // The URL password fills the unset field. + assert_eq!(credentials.password, "stale"); + + let no_credentials = merge_credentials( + &RtspVideoSourceConfig { username: None, password: None, ..overriding }, + &RtspUrl::parse("rtsp://camera.example/live").unwrap(), + ); + assert!(no_credentials.is_none()); + } +} From 366c5aaecdc6d676c94f5fa524d426e1b888e920 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:53 -0700 Subject: [PATCH 12/38] Document new source --- livekit-capture/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/livekit-capture/README.md b/livekit-capture/README.md index aa3541208..d61a3a095 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -57,6 +57,7 @@ named `source-`. Each module documents its source. | Feature | Source | Kind | | ------------------ | ---------------------- | ------- | | `source-gstreamer` | `GStreamerVideoSource` | encoded | +| `source-rtsp` | `RtspVideoSource` | encoded | | `source-pattern` | `PatternVideoSource` | pixel | | `source-clock` | `ClockVideoSource` | pixel | From fd12e9e4fd66877dc802870d47c0d65b5e8dbd60 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:46:54 -0700 Subject: [PATCH 13/38] Expose source over FFI --- livekit-ffi/Cargo.toml | 1 + livekit-ffi/protocol/capture.proto | 23 ++++++++++++++++++ livekit-ffi/src/conversion/capture.rs | 34 +++++++++++++++++++++++++-- livekit-ffi/src/server/capture.rs | 12 ++++++++++ 4 files changed, 68 insertions(+), 2 deletions(-) diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 1279518f7..eb18599e1 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -26,6 +26,7 @@ capture = ["dep:livekit-capture"] capture-clock = ["capture", "livekit-capture/source-clock"] capture-gstreamer = ["capture", "livekit-capture/source-gstreamer"] capture-pattern = ["capture", "livekit-capture/source-pattern"] +capture-rtsp = ["capture", "livekit-capture/source-rtsp"] [dependencies] livekit = { workspace = true } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 331d62dfa..30df94317 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -72,6 +72,28 @@ message GstreamerVideoSourceConfig { optional GstreamerRateControl rate_control = 4; } +// Encoded ingest from an RTSP server over TCP-interleaved RTP. +message RtspVideoSourceConfig { + // RTSP URL (rtsp://host[:port]/path). URL userinfo is accepted and + // stripped from requests; `username`/`password` take precedence over it. + required string url = 1; + // Username for RTSP authentication, overriding URL userinfo. + optional string username = 2; + // Password for RTSP authentication, overriding URL userinfo. + optional string password = 3; + // Codec required from the stream; the first supported video track offered + // by the SDP is used when omitted. + optional VideoCodec codec = 4; + // Encoded frame resolution. When omitted, it is discovered from the SDP + // when declared there, and from the stream's first keyframe otherwise; + // when set, the first keyframe is verified against it. + optional VideoSourceResolution resolution = 5; + // TCP connect and RTSP handshake timeout in milliseconds (default 10000). + optional uint32 connect_timeout_ms = 6; + // Maximum tolerated stream silence in milliseconds (default 30000). + optional uint32 idle_timeout_ms = 7; +} + // Test patterns built into livekit-capture. enum Pattern { // Animated color gradient. @@ -137,6 +159,7 @@ message NewCaptureSourceRequest { GstreamerVideoSourceConfig gstreamer = 1; PatternVideoSourceConfig pattern = 2; ClockVideoSourceConfig clock = 5; + RtspVideoSourceConfig rtsp = 6; } optional uint64 request_async_id = 3; } diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index 372d12754..0045ae4ba 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -15,7 +15,11 @@ use crate::proto; use livekit_capture::{encoded::EncodedVideoCodec, primitive::VideoResolution}; -#[cfg(any(feature = "capture-gstreamer", feature = "capture-pattern"))] +#[cfg(any( + feature = "capture-gstreamer", + feature = "capture-pattern", + feature = "capture-rtsp" +))] use crate::{FfiError, FfiResult}; #[cfg(feature = "capture-clock")] use livekit_capture::sources::clock::ClockVideoSourceConfig; @@ -25,6 +29,8 @@ use livekit_capture::sources::gstreamer::{ }; #[cfg(feature = "capture-pattern")] use livekit_capture::sources::pattern::{Pattern, PatternVideoSourceConfig}; +#[cfg(feature = "capture-rtsp")] +use livekit_capture::sources::rtsp::RtspVideoSourceConfig; impl From for VideoResolution { fn from(resolution: proto::VideoSourceResolution) -> Self { @@ -65,7 +71,7 @@ impl From for GStreamerBitrateUnit { } } -#[cfg(feature = "capture-gstreamer")] +#[cfg(any(feature = "capture-gstreamer", feature = "capture-rtsp"))] pub fn video_codec_from_proto(codec: proto::VideoCodec) -> EncodedVideoCodec { match codec { proto::VideoCodec::H264 => EncodedVideoCodec::H264, @@ -89,6 +95,30 @@ pub fn video_codec_to_proto(codec: EncodedVideoCodec) -> Option FfiResult { + let codec = config + .codec + .map(|value| { + proto::VideoCodec::try_from(value) + .map(video_codec_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid codec".into())) + }) + .transpose()?; + + Ok(RtspVideoSourceConfig { + url: config.url, + username: config.username, + password: config.password, + codec, + resolution: config.resolution.map(VideoResolution::from), + connect_timeout_ms: config.connect_timeout_ms, + idle_timeout_ms: config.idle_timeout_ms, + }) +} + #[cfg(feature = "capture-gstreamer")] pub fn gstreamer_config_from_proto( config: proto::GstreamerVideoSourceConfig, diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index e5e202ebd..abb74aacc 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -32,12 +32,16 @@ use livekit_capture::sources::clock::ClockVideoSource; use livekit_capture::sources::gstreamer::GStreamerVideoSource; #[cfg(feature = "capture-pattern")] use livekit_capture::sources::pattern::PatternVideoSource; +#[cfg(feature = "capture-rtsp")] +use livekit_capture::sources::rtsp::RtspVideoSource; use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; #[cfg(feature = "capture-gstreamer")] use crate::conversion::capture::gstreamer_config_from_proto; #[cfg(feature = "capture-pattern")] use crate::conversion::capture::pattern_config_from_proto; +#[cfg(feature = "capture-rtsp")] +use crate::conversion::capture::rtsp_config_from_proto; use crate::{conversion::capture::video_codec_to_proto, proto, FfiError, FfiHandleId, FfiResult}; /// A capture pump of either kind, boxed at the FFI edge. @@ -120,6 +124,14 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Encoded(EncodedVideoPump::new(source)) } + #[cfg(feature = "capture-rtsp")] + proto::new_capture_source_request::Config::Rtsp(config) => { + let source = RtspVideoSource::new(rtsp_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Encoded(EncodedVideoPump::new(source)) + } #[cfg(feature = "capture-pattern")] proto::new_capture_source_request::Config::Pattern(config) => { let source = PatternVideoSource::new(pattern_config_from_proto(config)?) From 4b4bf72590b2004bf18745c573efce8a356daf15 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 09:48:07 -0700 Subject: [PATCH 14/38] Changeset --- .changeset/capture-source-rtsp.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/capture-source-rtsp.md diff --git a/.changeset/capture-source-rtsp.md b/.changeset/capture-source-rtsp.md new file mode 100644 index 000000000..6b2e9f849 --- /dev/null +++ b/.changeset/capture-source-rtsp.md @@ -0,0 +1,6 @@ +--- +livekit-capture: minor +livekit-ffi: minor +--- + +Add a capture source that ingests encoded video from an RTSP server. From f5005d4bed70d994d1fb0d719056a24d167ff098 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:15:30 -0700 Subject: [PATCH 15/38] Add additional deps --- Cargo.lock | 129 +++++++++++++++++++++++++++++++++++++ livekit-capture/Cargo.toml | 17 +++++ 2 files changed, 146 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index bf24c3b9e..fa90d1b40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2437,6 +2437,23 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gio" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3e1f669909c326b9413bde5a742097b8c90a7d78f45326db13668984769ded" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "gio-sys 0.22.8", + "glib 0.22.8", + "libc", + "pin-project-lite", + "smallvec", +] + [[package]] name = "gio-sys" version = "0.21.5" @@ -2751,6 +2768,115 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gstreamer-net" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcad04d471a4f2c859ef1287f22581b07064e357cc89dfa415ffc99b2a3193d" +dependencies = [ + "gio", + "glib 0.22.8", + "gstreamer", + "gstreamer-net-sys", +] + +[[package]] +name = "gstreamer-net-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcefb342a98ffca0b106bc9fea13d5df15879e1613b4dc652a6ec1960c32584c" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-rtsp" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deeb8cb21db41f90402f4de49403f44492b737e872a06665b93d90987a1bacaa" +dependencies = [ + "glib 0.22.8", + "gstreamer", + "gstreamer-rtsp-sys", + "gstreamer-sdp", + "libc", +] + +[[package]] +name = "gstreamer-rtsp-server" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "379699a4451e4f73b1f19a11d6640c631d308bca130628161802d34e429a6c03" +dependencies = [ + "gio", + "glib 0.22.8", + "gstreamer", + "gstreamer-net", + "gstreamer-rtsp", + "gstreamer-rtsp-server-sys", + "gstreamer-sdp", + "libc", +] + +[[package]] +name = "gstreamer-rtsp-server-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca124fc3e4803fe31d618e29e53a66f46a442178c37303fccaea0f999366221c" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-net-sys", + "gstreamer-rtsp-sys", + "gstreamer-sdp-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-rtsp-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36b59f39a4577720af3ca5a4ca4402b8ee5ff96b391c8895d1b28ba481a3ad" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-sdp-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sdp" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a7859290ba340f1bd45d4bc0ada958700e9d3b1a522b456fd7ee88da468cd13" +dependencies = [ + "glib 0.22.8", + "gstreamer", + "gstreamer-sdp-sys", +] + +[[package]] +name = "gstreamer-sdp-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20f0eb41ecfbacbf6a29d9457e6de0f59e2638f47fbdb6a6c1bcfb720c2b9ee" +dependencies = [ + "glib-sys 0.22.8", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-sys" version = "0.25.2" @@ -3833,12 +3959,15 @@ dependencies = [ "chrono", "gstreamer", "gstreamer-app", + "gstreamer-rtsp", + "gstreamer-rtsp-server", "livekit", "log", "md-5", "pollster", "schemars", "serde", + "test-log", "thiserror 2.0.19", "tokio", "wgpu", diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 8f015585d..7519c88a1 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -13,6 +13,10 @@ bytes = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } +# Test-only (`__test-source-rtsp`); in `[dependencies]` because Cargo has no +# optional dev-dependencies. +gstreamer-rtsp = { version = "0.25", optional = true } +gstreamer-rtsp-server = { version = "0.25", optional = true } md-5 = { version = "0.10", optional = true } livekit = { workspace = true } log = { workspace = true } @@ -25,6 +29,8 @@ wgpu = { workspace = true, optional = true } yuv-sys = { workspace = true, features = ["jpeg"], optional = true } [dev-dependencies] +log = { workspace = true } +test-log = "0.2" tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] @@ -39,3 +45,14 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] source-rtsp = ["dep:base64", "dep:md-5"] + +# Testing +# In-process RTSP server for `tests/source_rtsp_test.rs`. Requires the +# system GStreamer RTSP server library. Test-only; not part of the public +# API. +__test-source-rtsp = [ + "source-rtsp", + "dep:gstreamer", + "dep:gstreamer-rtsp", + "dep:gstreamer-rtsp-server", +] From 77849a012a39ea197b1ef1125dabf5498f9cc6ff Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:15:31 -0700 Subject: [PATCH 16/38] Implement test server --- livekit-capture/tests/common/mod.rs | 64 +++++++++++ livekit-capture/tests/common/rtsp.rs | 162 +++++++++++++++++++++++++++ 2 files changed, 226 insertions(+) create mode 100644 livekit-capture/tests/common/mod.rs create mode 100644 livekit-capture/tests/common/rtsp.rs diff --git a/livekit-capture/tests/common/mod.rs b/livekit-capture/tests/common/mod.rs new file mode 100644 index 000000000..79b5acbd0 --- /dev/null +++ b/livekit-capture/tests/common/mod.rs @@ -0,0 +1,64 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared helpers for the livekit-capture integration tests. +//! +//! Source-specific helpers (test servers, pipelines) live in a submodule +//! per source, gated by that source's test feature. + +#[cfg(feature = "__test-source-rtsp")] +pub mod rtsp; + +use std::time::{Duration, Instant}; + +use livekit_capture::{ + encoded::{EncodedVideoSource, OwnedEncodedAccessUnit}, + pump::PumpStop, +}; + +/// How long one test waits for its access units before failing. +const PULL_DEADLINE: Duration = Duration::from_secs(15); + +/// Pulls `count` access units from an encoded source, failing the test if +/// the stream errors, ends, or stalls. +pub fn pull_access_units( + source: &mut impl EncodedVideoSource, + count: usize, +) -> Vec { + let stop = PumpStop::new(); + let deadline = Instant::now() + PULL_DEADLINE; + let mut access_units = Vec::with_capacity(count); + while access_units.len() < count { + assert!( + Instant::now() < deadline, + "timed out after {} of {count} access units", + access_units.len() + ); + match source.next_access_unit(&stop).expect("source failed") { + Some(access_unit) => { + log::debug!( + "pulled access unit {}/{count}: {:?} {:?}, {} bytes, ts {}us", + access_units.len() + 1, + access_unit.codec, + access_unit.frame_type, + access_unit.payload.len(), + access_unit.timestamp_us, + ); + access_units.push(access_unit); + } + None => panic!("stream ended after {} of {count} access units", access_units.len()), + } + } + access_units +} diff --git a/livekit-capture/tests/common/rtsp.rs b/livekit-capture/tests/common/rtsp.rs new file mode 100644 index 000000000..56dac2d2d --- /dev/null +++ b/livekit-capture/tests/common/rtsp.rs @@ -0,0 +1,162 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP test helpers: an in-process GStreamer RTSP server and the encoder +//! pipelines it serves. + +use std::{ + thread, + time::{Duration, Instant}, +}; + +use gstreamer::{self as gst, glib}; +use gstreamer_rtsp_server::{prelude::*, RTSPAuth, RTSPMediaFactory, RTSPServer, RTSPToken}; +use livekit_capture::sources::rtsp::RtspVideoSourceConfig; + +/// Test streams are 640x480 at 30 fps with a keyframe every 30 frames. +pub const TEST_WIDTH: u32 = 640; +pub const TEST_HEIGHT: u32 = 480; + +pub const H264_PIPELINE: &str = "videotestsrc is-live=true \ + ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ + ! x264enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + byte-stream=true aud=true \ + ! h264parse config-interval=-1 ! rtph264pay name=pay0 pt=96 config-interval=1"; + +pub const H265_PIPELINE: &str = "videotestsrc is-live=true \ + ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ + ! x265enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + option-string=repeat-headers=1:aud=1:open-gop=0 \ + ! h265parse config-interval=-1 ! rtph265pay name=pay0 pt=96 config-interval=1"; + +pub const VP8_PIPELINE: &str = "videotestsrc is-live=true \ + ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ + ! vp8enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500000 \ + ! rtpvp8pay name=pay0 pt=96"; + +/// An in-process GStreamer RTSP server serving one launch pipeline at +/// `/test` on an ephemeral localhost port. +pub struct RtspTestServer { + main_loop: glib::MainLoop, + thread: Option>, + port: i32, +} + +impl RtspTestServer { + /// Starts a server streaming `media_pipeline`, which must end in an RTP + /// payloader named `pay0`. + pub fn launch(media_pipeline: &str) -> Self { + Self::launch_inner(media_pipeline, None) + } + + /// Starts a server that requires Digest authentication. + pub fn launch_with_digest_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, Some((username, password))) + } + + fn launch_inner(media_pipeline: &str, digest: Option<(&str, &str)>) -> Self { + gst::init().expect("failed to initialize GStreamer"); + + // Each server runs on its own main context so parallel tests never + // contend for the default one. + let context = glib::MainContext::new(); + let server = RTSPServer::new(); + server.set_address("127.0.0.1"); + // Bind an ephemeral port; the real port is read back after attach. + server.set_service("0"); + + let factory = RTSPMediaFactory::new(); + factory.set_launch(&format!("( {media_pipeline} )")); + factory.set_shared(false); + + if let Some((username, password)) = digest { + factory.add_role_from_structure( + &gst::Structure::builder("user") + .field("media.factory.access", true) + .field("media.factory.construct", true) + .build(), + ); + let token = RTSPToken::builder().field("media.factory.role", "user").build(); + let auth = RTSPAuth::new(); + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); + auth.add_digest(username, password, &token); + server.set_auth(Some(&auth)); + } + + server + .mount_points() + .expect("RTSP server has no mount points") + .add_factory("/test", factory); + server.attach(Some(&context)).expect("failed to attach RTSP server"); + let port = server.bound_port(); + assert!(port > 0, "RTSP server reported no bound port"); + log::info!( + "RTSP test server listening at rtsp://127.0.0.1:{port}/test{}", + if digest.is_some() { " (digest auth)" } else { "" }, + ); + log::debug!("RTSP test server pipeline: {media_pipeline}"); + + let main_loop = glib::MainLoop::new(Some(&context), false); + let run_loop = main_loop.clone(); + let thread = thread::Builder::new() + .name("rtsp-test-server".to_owned()) + .spawn(move || run_loop.run()) + .expect("failed to spawn RTSP server thread"); + + // A quit before the loop runs would be lost; wait for it to start. + let started = Instant::now(); + while !main_loop.is_running() { + assert!( + started.elapsed() < Duration::from_secs(5), + "RTSP server main loop did not start" + ); + thread::sleep(Duration::from_millis(1)); + } + + Self { main_loop, thread: Some(thread), port } + } + + /// The stream's RTSP URL. + pub fn url(&self) -> String { + format!("rtsp://127.0.0.1:{}/test", self.port) + } +} + +impl Drop for RtspTestServer { + fn drop(&mut self) { + self.main_loop.quit(); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// A source configuration for `url` with test-friendly timeouts and every +/// optional field unset. +pub fn test_config(url: String) -> RtspVideoSourceConfig { + RtspVideoSourceConfig { + url, + username: None, + password: None, + codec: None, + resolution: None, + connect_timeout_ms: Some(5_000), + idle_timeout_ms: Some(10_000), + } +} From 9dac88c660ed8524726b2177e0f957394952c566 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:15:31 -0700 Subject: [PATCH 17/38] Implement integration tests --- livekit-capture/tests/source_rtsp_test.rs | 189 ++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100644 livekit-capture/tests/source_rtsp_test.rs diff --git a/livekit-capture/tests/source_rtsp_test.rs b/livekit-capture/tests/source_rtsp_test.rs new file mode 100644 index 000000000..218eace23 --- /dev/null +++ b/livekit-capture/tests/source_rtsp_test.rs @@ -0,0 +1,189 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for the RTSP source against an in-process GStreamer +//! RTSP server. Not run by default; see `tests/README.md`. + +#![cfg(feature = "__test-source-rtsp")] + +mod common; + +// Replaces `#[test]` with a variant that initializes logging, so `log::` +// output from the source and the test server is visible; see tests/README.md. +use test_log::test; + +use common::{ + pull_access_units, + rtsp::{ + test_config, RtspTestServer, H264_PIPELINE, H265_PIPELINE, TEST_HEIGHT, TEST_WIDTH, + VP8_PIPELINE, + }, +}; +use livekit_capture::{ + encoded::{h26x::annex_b_nalus, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource}, + primitive::VideoResolution, + sources::rtsp::{RtspVideoSource, RtspVideoSourceConfig}, +}; + +const TEST_RESOLUTION: VideoResolution = VideoResolution::new(TEST_WIDTH, TEST_HEIGHT); + +fn h264_nal_types(payload: &[u8]) -> Vec { + annex_b_nalus(payload).iter().map(|nal| nal[0] & 0x1f).collect() +} + +fn h265_nal_types(payload: &[u8]) -> Vec { + annex_b_nalus(payload).iter().map(|nal| (nal[0] >> 1) & 0x3f).collect() +} + +fn assert_increasing_timestamps(access_units: &[livekit_capture::encoded::OwnedEncodedAccessUnit]) { + for window in access_units.windows(2) { + assert!( + window[1].timestamp_us > window[0].timestamp_us, + "timestamps must increase: {} then {}", + window[0].timestamp_us, + window[1].timestamp_us, + ); + } +} + +#[test] +fn streams_h264_access_units() { + let server = RtspTestServer::launch(H264_PIPELINE); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + assert_eq!(source.codec(), EncodedVideoCodec::H264); + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 15); + + // The first access unit must be a self-contained keyframe: subscribers + // can only initialize a decoder from parameter sets inside it. + let first = &access_units[0]; + assert_eq!(first.frame_type, EncodedFrameType::Key); + let nal_types = h264_nal_types(&first.payload); + assert!(nal_types.contains(&7), "keyframe missing SPS: {nal_types:?}"); + assert!(nal_types.contains(&8), "keyframe missing PPS: {nal_types:?}"); + assert!(nal_types.contains(&5), "keyframe missing IDR: {nal_types:?}"); + + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert!(access_unit.payload.starts_with(&[0, 0, 0, 1])); + assert_eq!(access_unit.resolution, TEST_RESOLUTION); + } +} + +#[test] +fn discovers_h264_resolution() { + let server = RtspTestServer::launch(H264_PIPELINE); + let mut source = + RtspVideoSource::new_blocking(test_config(server.url())).expect("failed to connect"); + + assert_eq!(source.resolution(), TEST_RESOLUTION); + + // The keyframe consumed by discovery must not be lost. + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); + assert_eq!(first.resolution, TEST_RESOLUTION); +} + +#[test] +fn streams_h265_access_units() { + let server = RtspTestServer::launch(H265_PIPELINE); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H265), + ..test_config(server.url()) + }) + .expect("failed to connect"); + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + + // H.265 keyframes must carry VPS, SPS, and PPS alongside the IDR — via + // the stream or injected from the real server's SDP. + let first = &access_units[0]; + assert_eq!(first.frame_type, EncodedFrameType::Key); + let nal_types = h265_nal_types(&first.payload); + for parameter_set in [32u8, 33, 34] { + assert!(nal_types.contains(¶meter_set), "keyframe missing NAL {parameter_set}"); + } + assert!( + nal_types.iter().any(|nal_type| matches!(nal_type, 19 | 20)), + "keyframe missing IDR: {nal_types:?}" + ); + assert_increasing_timestamps(&access_units); +} + +#[test] +fn streams_vp8_access_units() { + let server = RtspTestServer::launch(VP8_PIPELINE); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP8), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // VP8 has no SDP resolution hints; discovery parses the first keyframe. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn rejects_codec_mismatch() { + let server = RtspTestServer::launch(H264_PIPELINE); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP8), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + assert!(err.to_string().contains("codec mismatch"), "unexpected error: {err}"); +} + +#[test] +fn authenticates_with_digest() { + let server = RtspTestServer::launch_with_digest_auth(H264_PIPELINE, "admin", "secret"); + + // Without credentials the server's challenge cannot be answered. + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + assert!(err.to_string().contains("credentials"), "unexpected error: {err}"); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect with credentials"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} From a84f77a4c08e52e67f6f63d154555eca5803e7a9 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 10:15:32 -0700 Subject: [PATCH 18/38] Document testing conventions --- livekit-capture/AGENTS.md | 22 ++++++++++++++++ livekit-capture/tests/README.md | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 livekit-capture/tests/README.md diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md index 19fa12189..745de9053 100644 --- a/livekit-capture/AGENTS.md +++ b/livekit-capture/AGENTS.md @@ -20,3 +20,25 @@ - NEVER add a source that is not consumable uniformly through one of these traits - Doing so would break API contract and break integration for consumers - Keep API surface minimal and hide implementation details + +## Adding integration tests + +- Integration tests verify a source against a real backend (e.g., a real + RTSP server, a real capture device) and never run by default +- Gate each source's tests behind an internal feature named + `__test-source-` (e.g., `__test-source-rtsp`) that enables + `source-` plus any test-only dependencies + - Test-only dependencies are optional entries in `[dependencies]` enabled + only by the test feature (Cargo has no optional dev-dependencies) +- Tests live in `tests/source__test.rs`, with + `#![cfg(feature = "__test-source-")]` at the top so the target + compiles to empty without the feature +- Helpers live in `tests/common/`: source-agnostic ones (e.g., pull loops) + in `tests/common/mod.rs`, and source-specific ones (test servers, + pipelines) in a `tests/common/.rs` submodule declared with + `#[cfg(feature = "__test-source-")]` +- Test the source through its public API (construct, `next_access_unit` or + `next_frame`); do not publish to an RTC source or a LiveKit server +- Launch backends in-process or automatically; a test must not depend on a + manually started process +- Document host prerequisites and the run command in `tests/README.md` diff --git a/livekit-capture/tests/README.md b/livekit-capture/tests/README.md new file mode 100644 index 000000000..78ff18cd5 --- /dev/null +++ b/livekit-capture/tests/README.md @@ -0,0 +1,46 @@ +# Integration Tests + +Integration tests verify capture sources against real backends, so they are +not enabled by default. Each source has an internal test feature named +`__test-source-` that builds and runs its tests in +`tests/source__test.rs`. Tests exercise the source directly through +its public API (construction and `next_access_unit`); they do not publish to +an RTC source and need no LiveKit server. + +Some sources can only be tested on hosts that provide their backend (for +example, a future `__test-source-device` needs a capture device), so each +section below documents its prerequisites. + +## RTSP (`__test-source-rtsp`) + +Each test starts an in-process GStreamer RTSP server on an ephemeral +localhost port and streams real encoded video to the source. The system +GStreamer installation must include the RTSP server library and the x264, +x265, and VP8 encoder plugins: + +- macOS: `brew install gstreamer` +- Debian/Ubuntu: `apt install libgstrtspserver-1.0-dev + gstreamer1.0-plugins-good gstreamer1.0-plugins-bad + gstreamer1.0-plugins-ugly` (in addition to the base development packages) + +```sh +cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test +``` + +## Logging + +Tests use the [`test-log`](https://crates.io/crates/test-log) `#[test]` +attribute, so `log::` output from the sources and the test helpers is +recorded per test. The standard test harness only prints it for *failing* +tests; to see it for passing tests, disable output capture: + +```sh +cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test -- --nocapture +``` + +That shows info-level logs (stream setup, discovered settings). For the +per-access-unit trace, raise the filter: + +```sh +RUST_LOG=debug cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test -- --nocapture +``` From 8b6108ac313a33590353fbd451448a68d84381ff Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:36:44 -0700 Subject: [PATCH 19/38] Initial RTSPs (secure) support --- Cargo.lock | 37 +++ livekit-capture/Cargo.toml | 10 + livekit-capture/README.md | 4 + livekit-capture/src/sources/rtsp/client.rs | 257 +++++++++++++++++++-- livekit-capture/src/sources/rtsp/mod.rs | 43 +++- livekit-capture/src/sources/rtsp/sdp.rs | 38 ++- livekit-capture/tests/README.md | 3 + livekit-capture/tests/common/rtsp.rs | 63 ++++- livekit-capture/tests/source_rtsp_test.rs | 52 +++++ livekit-ffi/Cargo.toml | 1 + livekit-ffi/protocol/capture.proto | 10 +- livekit-ffi/src/conversion/capture.rs | 1 + 12 files changed, 485 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fa90d1b40..745fd4c2c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3957,6 +3957,7 @@ dependencies = [ "base64 0.22.1", "bytes", "chrono", + "gio", "gstreamer", "gstreamer-app", "gstreamer-rtsp", @@ -3965,6 +3966,10 @@ dependencies = [ "log", "md-5", "pollster", + "rcgen", + "rustls", + "rustls-native-certs", + "rustls-pki-types", "schemars", "serde", "test-log", @@ -5544,6 +5549,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "peniko" version = "0.6.1" @@ -6288,6 +6303,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "read-fonts" version = "0.39.2" @@ -9523,6 +9551,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 7519c88a1..884b753a2 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -15,9 +15,13 @@ gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } # Test-only (`__test-source-rtsp`); in `[dependencies]` because Cargo has no # optional dev-dependencies. +gio = { version = "0.22", optional = true } gstreamer-rtsp = { version = "0.25", optional = true } gstreamer-rtsp-server = { version = "0.25", optional = true } md-5 = { version = "0.10", optional = true } +rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"], optional = true } +rustls-native-certs = { version = "0.8", optional = true } +rustls-pki-types = { version = "1", optional = true } livekit = { workspace = true } log = { workspace = true } pollster = { version = "0.4", optional = true } @@ -30,6 +34,7 @@ yuv-sys = { workspace = true, features = ["jpeg"], optional = true } [dev-dependencies] log = { workspace = true } +rcgen = "0.13" test-log = "0.2" tokio = { workspace = true, features = ["rt", "time", "macros"] } @@ -45,6 +50,9 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] source-rtsp = ["dep:base64", "dep:md-5"] +# Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. +# Certificates are verified against the system roots by default. +source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] # Testing # In-process RTSP server for `tests/source_rtsp_test.rs`. Requires the @@ -52,6 +60,8 @@ source-rtsp = ["dep:base64", "dep:md-5"] # API. __test-source-rtsp = [ "source-rtsp", + "source-rtsp-tls", + "dep:gio", "dep:gstreamer", "dep:gstreamer-rtsp", "dep:gstreamer-rtsp-server", diff --git a/livekit-capture/README.md b/livekit-capture/README.md index d61a3a095..d60db3203 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -61,6 +61,10 @@ named `source-`. Each module documents its source. | `source-pattern` | `PatternVideoSource` | pixel | | `source-clock` | `ClockVideoSource` | pixel | +`source-rtsp-tls` extends `RtspVideoSource` with `rtsps://` support (RTSP +over TLS 1.2+). Certificates are verified against the system roots by +default; cameras with self-signed certificates can opt out per source. + ## Custom sources Implement `pixel::PixelVideoSource` for raw frames or diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index 47e309de4..a61b223bc 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -41,7 +41,7 @@ const MAX_HEADER_BYTES: usize = 64 * 1024; /// Bytes requested from the socket per read. const READ_CHUNK_BYTES: usize = 8 * 1024; -/// A parsed `rtsp://` URL. +/// A parsed `rtsp://` or `rtsps://` URL. #[derive(Debug, Clone, PartialEq, Eq)] pub(super) struct RtspUrl { /// Request URI with any userinfo stripped, so credentials never appear @@ -51,15 +51,21 @@ pub(super) struct RtspUrl { pub(super) host_header: String, /// Credentials from the URL userinfo, percent-decoded. pub(super) credentials: Option, + /// Whether the URL requires TLS (`rtsps://`). + pub(super) tls: bool, connect_host: String, port: u16, } impl RtspUrl { - /// Parses an `rtsp://[user:password@]host[:port][/path]` URL. + /// Parses an `rtsp(s)://[user:password@]host[:port][/path]` URL. pub(super) fn parse(url: &str) -> Result { - let Some(rest) = url.strip_prefix("rtsp://") else { - return Err(RtspVideoSourceError::InvalidUrl("expected rtsp:// scheme")); + let (scheme, rest, tls) = if let Some(rest) = url.strip_prefix("rtsp://") { + ("rtsp://", rest, false) + } else if let Some(rest) = url.strip_prefix("rtsps://") { + ("rtsps://", rest, true) + } else { + return Err(RtspVideoSourceError::InvalidUrl("expected rtsp:// or rtsps:// scheme")); }; let (authority, path_suffix) = match rest.find('/') { Some(path_start) => (&rest[..path_start], &rest[path_start..]), @@ -73,7 +79,8 @@ impl RtspUrl { if host_port.is_empty() { return Err(RtspVideoSourceError::InvalidUrl("missing host")); } - let (connect_host, port) = parse_host_port(host_port)?; + let default_port = if tls { 322 } else { 554 }; + let (connect_host, port) = parse_host_port(host_port, default_port)?; let host_header = if host_port.contains(':') { host_port.to_owned() } else { @@ -81,9 +88,10 @@ impl RtspUrl { }; Ok(Self { - request_uri: format!("rtsp://{host_port}{path_suffix}"), + request_uri: format!("{scheme}{host_port}{path_suffix}"), host_header, credentials, + tls, connect_host, port, }) @@ -123,12 +131,15 @@ fn percent_decode(value: &str) -> String { String::from_utf8_lossy(&decoded).into_owned() } -fn parse_host_port(host_port: &str) -> Result<(String, u16), RtspVideoSourceError> { +fn parse_host_port( + host_port: &str, + default_port: u16, +) -> Result<(String, u16), RtspVideoSourceError> { if let Some(rest) = host_port.strip_prefix('[') { let Some((host, after_host)) = rest.split_once(']') else { return Err(RtspVideoSourceError::InvalidUrl("malformed IPv6 host")); }; - let port = after_host.strip_prefix(':').map(parse_port).transpose()?.unwrap_or(554); + let port = after_host.strip_prefix(':').map(parse_port).transpose()?.unwrap_or(default_port); return Ok((host.to_owned(), port)); } @@ -138,7 +149,7 @@ fn parse_host_port(host_port: &str) -> Result<(String, u16), RtspVideoSourceErro } } - Ok((host_port.to_owned(), 554)) + Ok((host_port.to_owned(), default_port)) } fn parse_port(port: &str) -> Result { @@ -201,10 +212,56 @@ enum StreamFill { TimedOut, } -/// RTSP connection: owns the TCP stream, the read buffer, the request +/// The connection's byte stream: plain TCP, or TLS over TCP for `rtsps://`. +enum Transport { + Plain(TcpStream), + #[cfg(feature = "source-rtsp-tls")] + Tls(Box>), +} + +impl Transport { + /// The underlying TCP socket, whose timeouts bound every read and write. + fn socket(&self) -> &TcpStream { + match self { + Self::Plain(stream) => stream, + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.get_ref(), + } + } +} + +impl Read for Transport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Plain(stream) => stream.read(buf), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.read(buf), + } + } +} + +impl Write for Transport { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Plain(stream) => stream.write(buf), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Plain(stream) => stream.flush(), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.flush(), + } + } +} + +/// RTSP connection: owns the transport stream, the read buffer, the request /// sequence number, and the authentication context. pub(super) struct RtspClient { - stream: TcpStream, + stream: Transport, buf: BytesMut, scratch: Vec, cseq: u32, @@ -225,13 +282,17 @@ impl fmt::Debug for RtspClient { } impl RtspClient { - /// Connects to the URL's host, bounded by `deadline`, and prepares the - /// socket for polled reads. + /// Connects to the URL's host, bounded by `deadline`, establishes TLS + /// for `rtsps://` URLs, and prepares the socket for polled reads. pub(super) fn connect( url: &RtspUrl, credentials: Option, + accept_invalid_tls_certs: bool, deadline: Instant, ) -> Result { + #[cfg(not(feature = "source-rtsp-tls"))] + let _ = accept_invalid_tls_certs; + let addrs = (url.connect_host.as_str(), url.port).to_socket_addrs()?; let mut last_error = None; let mut stream = None; @@ -260,6 +321,19 @@ impl RtspClient { stream.set_read_timeout(Some(READ_POLL))?; stream.set_write_timeout(Some(WRITE_TIMEOUT))?; + let stream = match url.tls { + false => Transport::Plain(stream), + #[cfg(feature = "source-rtsp-tls")] + true => Transport::Tls(Box::new(tls::establish( + stream, + &url.connect_host, + accept_invalid_tls_certs, + deadline, + )?)), + #[cfg(not(feature = "source-rtsp-tls"))] + true => return Err(RtspVideoSourceError::TlsNotSupported), + }; + Ok(Self { stream, buf: BytesMut::with_capacity(READ_CHUNK_BYTES), @@ -427,9 +501,149 @@ impl RtspClient { } Err(err) if err.kind() == io::ErrorKind::Interrupted => {} Err(err) if is_timeout_io_error(&err) => return Ok(StreamFill::TimedOut), + // TLS peers that drop the connection without a close_notify + // (most cameras) surface as UnexpectedEof; treat it like a + // plain EOF and let the caller decide whether the framing + // was left mid-unit. + Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => { + return Ok(StreamFill::Eof) + } + Err(err) => return Err(err.into()), + } + } + } +} + +/// TLS support for `rtsps://` URLs. +#[cfg(feature = "source-rtsp-tls")] +mod tls { + use std::{io, net::TcpStream, sync::Arc, time::Instant}; + + use rustls::{ClientConfig, ClientConnection, RootCertStore, StreamOwned}; + use rustls_pki_types::ServerName; + + use super::{is_timeout_io_error, RtspPhase, RtspVideoSourceError}; + + /// Establishes TLS over a connected TCP stream, driving the handshake to + /// completion bounded by `deadline`. + /// + /// The handshake must finish here: afterwards, writes never need to + /// read, so the request path's timeout handling stays valid. During the + /// handshake, the socket's read timeout is the retry granularity. + pub(super) fn establish( + stream: TcpStream, + host: &str, + accept_invalid_certs: bool, + deadline: Instant, + ) -> Result, RtspVideoSourceError> { + let config = client_config(accept_invalid_certs)?; + // `ServerName` accepts both DNS names and the IP literals cameras + // are usually addressed by. + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|err| RtspVideoSourceError::Tls(err.to_string()))?; + let mut connection = ClientConnection::new(Arc::new(config), server_name) + .map_err(|err| RtspVideoSourceError::Tls(err.to_string()))?; + + let mut stream = stream; + while connection.is_handshaking() { + match connection.complete_io(&mut stream) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) if is_timeout_io_error(&err) => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::Timeout { phase: RtspPhase::Connect }); + } + } + // rustls reports TLS-level handshake failures as InvalidData. + Err(err) if err.kind() == io::ErrorKind::InvalidData => { + return Err(RtspVideoSourceError::Tls(err.to_string())); + } Err(err) => return Err(err.into()), } } + Ok(StreamOwned::new(connection, stream)) + } + + fn client_config(accept_invalid_certs: bool) -> Result { + if accept_invalid_certs { + return Ok(ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(danger::NoVerification)) + .with_no_client_auth()); + } + + let mut roots = RootCertStore::empty(); + // Individually unparsable certificates in the OS store are skipped; + // an entirely unavailable store is an error. + let native = rustls_native_certs::load_native_certs(); + for cert in native.certs { + let _ = roots.add(cert); + } + if roots.is_empty() { + return Err(RtspVideoSourceError::Tls( + "no usable system root certificates".to_owned(), + )); + } + Ok(ClientConfig::builder().with_root_certificates(roots).with_no_client_auth()) + } + + /// Certificate "verification" that accepts anything; see + /// [`RtspVideoSourceConfig::accept_invalid_tls_certs`](super::super::RtspVideoSourceConfig::accept_invalid_tls_certs). + mod danger { + use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + crypto::{ring, verify_tls12_signature, verify_tls13_signature}, + DigitallySignedStruct, Error, SignatureScheme, + }; + use rustls_pki_types::{CertificateDer, ServerName, UnixTime}; + + #[derive(Debug)] + pub(super) struct NoVerification; + + impl ServerCertVerifier for NoVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls12_signature( + message, + cert, + dss, + &ring::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls13_signature( + message, + cert, + dss, + &ring::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + ring::default_provider().signature_verification_algorithms.supported_schemes() + } + } } } @@ -561,6 +775,23 @@ mod tests { ); } + #[test] + fn parses_rtsps_urls() { + let url = RtspUrl::parse("rtsps://camera.example/live").unwrap(); + assert!(url.tls); + assert_eq!(url.port, 322); + assert_eq!(url.host_header, "camera.example:322"); + assert_eq!(url.request_uri, "rtsps://camera.example/live"); + + let url = RtspUrl::parse("rtsps://admin:secret@camera.example:7441/live").unwrap(); + assert!(url.tls); + assert_eq!(url.port, 7441); + assert_eq!(url.request_uri, "rtsps://camera.example:7441/live"); + assert!(url.credentials.is_some()); + + assert!(!RtspUrl::parse("rtsp://camera.example/live").unwrap().tls); + } + #[test] fn defaults_to_port_554() { let url = RtspUrl::parse("rtsp://camera.example/live").unwrap(); diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index 6ac55478c..4595e3863 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -19,6 +19,11 @@ //! re-encoding. Basic and Digest authentication are supported, and packet //! loss is recovered by waiting for the next keyframe. //! +//! With the `source-rtsp-tls` feature, `rtsps://` URLs are supported: the +//! whole connection — control and interleaved media — runs over TLS (1.2 or +//! newer). Certificates are verified against the system roots unless +//! [`RtspVideoSourceConfig::accept_invalid_tls_certs`] opts out. +//! //! The connection is not re-established on failure: a connection error ends //! the source with an error, and a clean server-side end of stream ends it //! like any finite source. @@ -72,7 +77,8 @@ const DEFAULT_SESSION_TIMEOUT_SECS: u64 = 60; )] #[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] pub struct RtspVideoSourceConfig { - /// RTSP URL (`rtsp://host[:port]/path`). + /// RTSP URL (`rtsp://host[:port]/path`, or `rtsps://` with the + /// `source-rtsp-tls` feature). /// /// URL userinfo (`rtsp://user:password@...`) is accepted and stripped /// from requests; [`Self::username`] and [`Self::password`] take @@ -112,6 +118,15 @@ pub struct RtspVideoSourceConfig { /// limit. #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] pub idle_timeout_ms: Option, + + /// Disables TLS certificate verification for `rtsps://` URLs. + /// + /// Most cameras present self-signed certificates, which fail + /// verification against the system roots. Enabling this accepts any + /// certificate: the connection is still encrypted, but not + /// authenticated — a network attacker could impersonate the camera. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "std::ops::Not::not"))] + pub accept_invalid_tls_certs: bool, } /// Protocol phase a timeout occurred in. @@ -201,6 +216,12 @@ pub enum RtspVideoSourceError { /// Interleaved framing was malformed or a non-interleaved byte arrived. #[error("unexpected RTSP interleaved data")] UnexpectedData, + /// The URL requires TLS but the `source-rtsp-tls` feature is disabled. + #[error("rtsps:// requires the `source-rtsp-tls` feature, which is not enabled")] + TlsNotSupported, + /// TLS configuration or the TLS handshake failed. + #[error("RTSP TLS failed: {0}")] + Tls(String), /// The stream produced no keyframe during resolution discovery. #[error( "stream produced no keyframe during resolution discovery; declare `resolution` in the \ @@ -297,12 +318,21 @@ impl RtspVideoSource { fn connect(config: RtspVideoSourceConfig) -> Result { let url = RtspUrl::parse(&config.url)?; + #[cfg(not(feature = "source-rtsp-tls"))] + if url.tls { + return Err(RtspVideoSourceError::TlsNotSupported); + } let credentials = merge_credentials(&config, &url); let connect_timeout = duration_ms(config.connect_timeout_ms, DEFAULT_CONNECT_TIMEOUT); let idle_timeout = duration_ms(config.idle_timeout_ms, DEFAULT_IDLE_TIMEOUT); let handshake_deadline = Instant::now() + connect_timeout; - let mut client = RtspClient::connect(&url, credentials, handshake_deadline)?; + let mut client = RtspClient::connect( + &url, + credentials, + config.accept_invalid_tls_certs, + handshake_deadline, + )?; let describe = client.request( "DESCRIBE", @@ -614,9 +644,18 @@ a=rtpmap:96 VP8/90000\r\n"; resolution: Some(VideoResolution::new(640, 480)), connect_timeout_ms: Some(2_000), idle_timeout_ms: None, + accept_invalid_tls_certs: false, } } + #[cfg(not(feature = "source-rtsp-tls"))] + #[test] + fn rejects_rtsps_url_without_tls_feature() { + let err = RtspVideoSource::new_blocking(config("rtsps://127.0.0.1:322/camera".to_owned())) + .unwrap_err(); + assert!(err.to_string().contains("source-rtsp-tls"), "unexpected error: {err}"); + } + fn rtp_packet(sequence_number: u16, timestamp: u32, marker: bool, payload: &[u8]) -> Vec { let mut packet = Vec::with_capacity(12 + payload.len()); packet.push(0x80); diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs index 368820b1e..97d5760cb 100644 --- a/livekit-capture/src/sources/rtsp/sdp.rs +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -271,6 +271,14 @@ fn decode_base64_nals(value: &str) -> Vec> { .collect() } +/// Splits an `rtsp://` or `rtsps://` URL into its scheme and remainder. +fn split_rtsp_scheme(url: &str) -> Option<(&str, &str)> { + if let Some(rest) = url.strip_prefix("rtsp://") { + return Some(("rtsp://", rest)); + } + url.strip_prefix("rtsps://").map(|rest| ("rtsps://", rest)) +} + /// Resolves an SDP `control` attribute against the session base URL. fn resolve_control_url(base_url: &str, control: Option<&str>) -> String { let Some(control) = control.map(str::trim).filter(|control| !control.is_empty()) else { @@ -279,15 +287,14 @@ fn resolve_control_url(base_url: &str, control: Option<&str>) -> String { if control == "*" { return base_url.to_owned(); } - if control.starts_with("rtsp://") { + if split_rtsp_scheme(control).is_some() { return control.to_owned(); } if control.starts_with('/') { - let authority = base_url - .strip_prefix("rtsp://") - .map(|rest| rest.split('/').next().unwrap_or(rest)) - .unwrap_or_default(); - return format!("rtsp://{authority}{control}"); + // An absolute path keeps the base URL's scheme and authority. + let (scheme, rest) = split_rtsp_scheme(base_url).unwrap_or(("rtsp://", base_url)); + let authority = rest.split('/').next().unwrap_or(rest); + return format!("{scheme}{authority}{control}"); } format!("{}/{}", base_url.trim_end_matches('/'), control) } @@ -524,6 +531,25 @@ a=rtpmap:96 H264/90000\r\n"; ); } + #[test] + fn resolves_control_urls_with_rtsps_scheme() { + // An absolute path keeps the rtsps scheme of the base URL. + assert_eq!( + resolve_control_url("rtsps://camera.example:7441/live", Some("/stream/trackID=1")), + "rtsps://camera.example:7441/stream/trackID=1" + ); + // A relative control appends to the rtsps base. + assert_eq!( + resolve_control_url("rtsps://camera.example:7441/live", Some("trackID=1")), + "rtsps://camera.example:7441/live/trackID=1" + ); + // An absolute rtsps URL passes through. + assert_eq!( + resolve_control_url(BASE_URL, Some("rtsps://camera.example/other")), + "rtsps://camera.example/other" + ); + } + #[test] fn resolves_control_against_content_base() { let sdp = "\ diff --git a/livekit-capture/tests/README.md b/livekit-capture/tests/README.md index 78ff18cd5..397fddd5e 100644 --- a/livekit-capture/tests/README.md +++ b/livekit-capture/tests/README.md @@ -27,6 +27,9 @@ x265, and VP8 encoder plugins: cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test ``` +The RTSPS tests generate a self-signed certificate at run time (via +`rcgen`), so TLS needs no host setup beyond the GStreamer packages above. + ## Logging Tests use the [`test-log`](https://crates.io/crates/test-log) `#[test]` diff --git a/livekit-capture/tests/common/rtsp.rs b/livekit-capture/tests/common/rtsp.rs index 56dac2d2d..65b414f68 100644 --- a/livekit-capture/tests/common/rtsp.rs +++ b/livekit-capture/tests/common/rtsp.rs @@ -52,13 +52,14 @@ pub struct RtspTestServer { main_loop: glib::MainLoop, thread: Option>, port: i32, + tls: bool, } impl RtspTestServer { /// Starts a server streaming `media_pipeline`, which must end in an RTP /// payloader named `pay0`. pub fn launch(media_pipeline: &str) -> Self { - Self::launch_inner(media_pipeline, None) + Self::launch_inner(media_pipeline, None, false) } /// Starts a server that requires Digest authentication. @@ -67,10 +68,25 @@ impl RtspTestServer { username: &str, password: &str, ) -> Self { - Self::launch_inner(media_pipeline, Some((username, password))) + Self::launch_inner(media_pipeline, Some((username, password)), false) } - fn launch_inner(media_pipeline: &str, digest: Option<(&str, &str)>) -> Self { + /// Starts a server that requires TLS (`rtsps://`), presenting a + /// freshly generated self-signed certificate. + pub fn launch_tls(media_pipeline: &str) -> Self { + Self::launch_inner(media_pipeline, None, true) + } + + /// Starts a server that requires both TLS and Digest authentication. + pub fn launch_tls_with_digest_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, Some((username, password)), true) + } + + fn launch_inner(media_pipeline: &str, digest: Option<(&str, &str)>, tls: bool) -> Self { gst::init().expect("failed to initialize GStreamer"); // Each server runs on its own main context so parallel tests never @@ -85,17 +101,28 @@ impl RtspTestServer { factory.set_launch(&format!("( {media_pipeline} )")); factory.set_shared(false); - if let Some((username, password)) = digest { + if digest.is_some() || tls { factory.add_role_from_structure( &gst::Structure::builder("user") .field("media.factory.access", true) .field("media.factory.construct", true) .build(), ); - let token = RTSPToken::builder().field("media.factory.role", "user").build(); let auth = RTSPAuth::new(); - auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); - auth.add_digest(username, password, &token); + if tls { + // Once a certificate is set, gst-rtsp-server requires TLS on + // every connection to this server instance. + auth.set_tls_certificate(Some(&self_signed_certificate())); + } + if let Some((username, password)) = digest { + let token = RTSPToken::builder().field("media.factory.role", "user").build(); + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); + auth.add_digest(username, password, &token); + } else { + // TLS without authentication: admit anonymous clients. + let mut token = RTSPToken::builder().field("media.factory.role", "user").build(); + auth.set_default_token(Some(&mut token)); + } server.set_auth(Some(&auth)); } @@ -107,7 +134,8 @@ impl RtspTestServer { let port = server.bound_port(); assert!(port > 0, "RTSP server reported no bound port"); log::info!( - "RTSP test server listening at rtsp://127.0.0.1:{port}/test{}", + "RTSP test server listening at {}://127.0.0.1:{port}/test{}", + if tls { "rtsps" } else { "rtsp" }, if digest.is_some() { " (digest auth)" } else { "" }, ); log::debug!("RTSP test server pipeline: {media_pipeline}"); @@ -129,15 +157,27 @@ impl RtspTestServer { thread::sleep(Duration::from_millis(1)); } - Self { main_loop, thread: Some(thread), port } + Self { main_loop, thread: Some(thread), port, tls } } - /// The stream's RTSP URL. + /// The stream's RTSP or RTSPS URL. pub fn url(&self) -> String { - format!("rtsp://127.0.0.1:{}/test", self.port) + let scheme = if self.tls { "rtsps" } else { "rtsp" }; + format!("{scheme}://127.0.0.1:{}/test", self.port) } } +/// Generates a fresh self-signed certificate for the test server. +fn self_signed_certificate() -> gio::TlsCertificate { + let certified = rcgen::generate_simple_self_signed(vec![ + "localhost".to_owned(), + "127.0.0.1".to_owned(), + ]) + .expect("failed to generate a self-signed certificate"); + let pem = format!("{}{}", certified.cert.pem(), certified.key_pair.serialize_pem()); + gio::TlsCertificate::from_pem(&pem).expect("failed to load the certificate into GIO") +} + impl Drop for RtspTestServer { fn drop(&mut self) { self.main_loop.quit(); @@ -158,5 +198,6 @@ pub fn test_config(url: String) -> RtspVideoSourceConfig { resolution: None, connect_timeout_ms: Some(5_000), idle_timeout_ms: Some(10_000), + accept_invalid_tls_certs: false, } } diff --git a/livekit-capture/tests/source_rtsp_test.rs b/livekit-capture/tests/source_rtsp_test.rs index 218eace23..1406da9f3 100644 --- a/livekit-capture/tests/source_rtsp_test.rs +++ b/livekit-capture/tests/source_rtsp_test.rs @@ -163,6 +163,58 @@ fn rejects_codec_mismatch() { assert!(err.to_string().contains("codec mismatch"), "unexpected error: {err}"); } +#[test] +fn streams_h264_over_rtsps() { + let server = RtspTestServer::launch_tls(H264_PIPELINE); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + // The test server's certificate is self-signed. + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect over TLS"); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); +} + +#[test] +fn rejects_untrusted_tls_certificate() { + let server = RtspTestServer::launch_tls(H264_PIPELINE); + // Default configuration verifies against the system roots, which must + // reject the server's self-signed certificate. + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + let message = err.to_string(); + assert!( + message.contains("TLS") || message.contains("certificate"), + "unexpected error: {message}" + ); +} + +#[test] +fn authenticates_with_digest_over_rtsps() { + let server = RtspTestServer::launch_tls_with_digest_auth(H264_PIPELINE, "admin", "secret"); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect with credentials over TLS"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} + #[test] fn authenticates_with_digest() { let server = RtspTestServer::launch_with_digest_auth(H264_PIPELINE, "admin", "secret"); diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index eb18599e1..1ae45c283 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -27,6 +27,7 @@ capture-clock = ["capture", "livekit-capture/source-clock"] capture-gstreamer = ["capture", "livekit-capture/source-gstreamer"] capture-pattern = ["capture", "livekit-capture/source-pattern"] capture-rtsp = ["capture", "livekit-capture/source-rtsp"] +capture-rtsp-tls = ["capture-rtsp", "livekit-capture/source-rtsp-tls"] [dependencies] livekit = { workspace = true } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 30df94317..b4ddadc68 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -74,8 +74,9 @@ message GstreamerVideoSourceConfig { // Encoded ingest from an RTSP server over TCP-interleaved RTP. message RtspVideoSourceConfig { - // RTSP URL (rtsp://host[:port]/path). URL userinfo is accepted and - // stripped from requests; `username`/`password` take precedence over it. + // RTSP URL (rtsp://host[:port]/path, or rtsps:// when the server is built + // with TLS support). URL userinfo is accepted and stripped from requests; + // `username`/`password` take precedence over it. required string url = 1; // Username for RTSP authentication, overriding URL userinfo. optional string username = 2; @@ -92,6 +93,11 @@ message RtspVideoSourceConfig { optional uint32 connect_timeout_ms = 6; // Maximum tolerated stream silence in milliseconds (default 30000). optional uint32 idle_timeout_ms = 7; + // Disables TLS certificate verification for rtsps:// URLs. Most cameras + // present self-signed certificates, which fail verification against the + // system roots. The connection stays encrypted but is not authenticated: + // a network attacker could impersonate the camera. + optional bool accept_invalid_tls_certs = 8; } // Test patterns built into livekit-capture. diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index 0045ae4ba..5bcb3071e 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -116,6 +116,7 @@ pub fn rtsp_config_from_proto( resolution: config.resolution.map(VideoResolution::from), connect_timeout_ms: config.connect_timeout_ms, idle_timeout_ms: config.idle_timeout_ms, + accept_invalid_tls_certs: config.accept_invalid_tls_certs.unwrap_or_default(), }) } From 76bc90505890fbe38ad1c7de294a7e588b3876f3 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:18:20 -0700 Subject: [PATCH 20/38] Use http-auth crate --- Cargo.lock | 23 +- livekit-capture/Cargo.toml | 4 +- livekit-capture/src/sources/rtsp/auth.rs | 375 +++++---------------- livekit-capture/src/sources/rtsp/client.rs | 8 + livekit-capture/src/sources/rtsp/mod.rs | 13 +- 5 files changed, 115 insertions(+), 308 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 745fd4c2c..3e3f39096 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3017,6 +3017,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hexf-parse" version = "0.2.1" @@ -3068,6 +3074,21 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "base64 0.22.1", + "digest", + "hex", + "md-5", + "memchr", + "rand 0.8.7", + "sha2", +] + [[package]] name = "http-body" version = "0.4.6" @@ -3962,9 +3983,9 @@ dependencies = [ "gstreamer-app", "gstreamer-rtsp", "gstreamer-rtsp-server", + "http-auth", "livekit", "log", - "md-5", "pollster", "rcgen", "rustls", diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 884b753a2..f4a297990 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -18,7 +18,7 @@ gstreamer-app = { version = "0.25.2", optional = true } gio = { version = "0.22", optional = true } gstreamer-rtsp = { version = "0.25", optional = true } gstreamer-rtsp-server = { version = "0.25", optional = true } -md-5 = { version = "0.10", optional = true } +http-auth = { version = "0.1", optional = true } rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"], optional = true } rustls-native-certs = { version = "0.8", optional = true } rustls-pki-types = { version = "1", optional = true } @@ -49,7 +49,7 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] -source-rtsp = ["dep:base64", "dep:md-5"] +source-rtsp = ["dep:base64", "dep:http-auth"] # Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. # Certificates are verified against the system roots by default. source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] diff --git a/livekit-capture/src/sources/rtsp/auth.rs b/livekit-capture/src/sources/rtsp/auth.rs index 82804185d..127ed1a4b 100644 --- a/livekit-capture/src/sources/rtsp/auth.rs +++ b/livekit-capture/src/sources/rtsp/auth.rs @@ -12,16 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! RTSP authentication: Basic and Digest (MD5 with `qop=auth`). +//! RTSP authentication, backed by the `http-auth` crate: Basic and Digest +//! (RFC 7616, including SHA-256 and session variants). -use std::{ - collections::hash_map::RandomState, - fmt, - hash::{BuildHasher, Hasher}, -}; - -use base64::{engine::general_purpose, Engine as _}; -use md5::{Digest, Md5}; +use std::fmt; use super::{client::RtspResponse, RtspVideoSourceError}; @@ -43,19 +37,16 @@ impl fmt::Debug for RtspCredentials { } /// Tracks the server's authentication challenge across requests. -#[derive(Debug, Clone)] pub(super) struct RtspAuthContext { credentials: Option, - challenge: Option, - nonce_count: u32, - cnonce: String, + client: Option, } impl RtspAuthContext { /// Creates an authentication context; without credentials, a challenge /// fails with [`RtspVideoSourceError::MissingCredentials`]. pub(super) fn new(credentials: Option) -> Self { - Self { credentials, challenge: None, nonce_count: 0, cnonce: make_cnonce() } + Self { credentials, client: None } } /// Builds the `Authorization` header value for a request, once the server @@ -65,32 +56,26 @@ impl RtspAuthContext { method: &str, uri: &str, ) -> Result, RtspVideoSourceError> { - let Some(challenge) = self.challenge.clone() else { + let Some(client) = self.client.as_mut() else { return Ok(None); }; let credentials = self.credentials.as_ref().ok_or(RtspVideoSourceError::MissingCredentials)?; - match challenge { - RtspAuthChallenge::Basic => { - let token = general_purpose::STANDARD - .encode(format!("{}:{}", credentials.username, credentials.password)); - Ok(Some(format!("Basic {token}"))) - } - RtspAuthChallenge::Digest(challenge) => { - self.nonce_count = self.nonce_count.saturating_add(1); - Ok(Some(build_digest_authorization( - credentials, - &challenge, - method, - uri, - self.nonce_count, - &self.cnonce, - ))) - } - } + let authorization = client + .respond(&http_auth::PasswordParams { + username: &credentials.username, + password: &credentials.password, + uri, + method, + // RTSP requests carry no body relevant to `auth-int`. + body: Some(&[]), + }) + .map_err(RtspVideoSourceError::Auth)?; + Ok(Some(authorization)) } - /// Ingests the challenge of a 401 response so the retry can authenticate. + /// Ingests the challenges of a 401 response so the retry can + /// authenticate. Digest is preferred over Basic when both are offered. pub(super) fn update_from_unauthorized( &mut self, response: &RtspResponse, @@ -98,291 +83,94 @@ impl RtspAuthContext { if self.credentials.is_none() { return Err(RtspVideoSourceError::MissingCredentials); } - self.challenge = Some(parse_authenticate_header( - response.headers("www-authenticate").collect::>().as_slice(), - )?); - self.nonce_count = 0; - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum RtspAuthChallenge { - Basic, - Digest(DigestAuthChallenge), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -struct DigestAuthChallenge { - realm: String, - nonce: String, - opaque: Option, - qop: Option, -} - -fn parse_authenticate_header( - headers: &[&str], -) -> Result { - for header in headers { - if strip_auth_scheme(header, "Digest").is_some() { - return parse_digest_challenge(header); - } - } - for header in headers { - if strip_auth_scheme(header, "Basic").is_some() { - return Ok(RtspAuthChallenge::Basic); - } - } - let scheme = headers - .first() - .and_then(|header| header.split_whitespace().next()) - .unwrap_or_default() - .to_owned(); - Err(RtspVideoSourceError::UnsupportedAuthScheme(scheme)) -} - -fn parse_digest_challenge(header: &str) -> Result { - let params = parse_auth_params( - strip_auth_scheme(header, "Digest").ok_or(RtspVideoSourceError::InvalidAuthChallenge)?, - ); - let realm = params - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("realm")) - .map(|(_, value)| value.to_owned()) - .ok_or(RtspVideoSourceError::InvalidAuthChallenge)?; - let nonce = params - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("nonce")) - .map(|(_, value)| value.to_owned()) - .ok_or(RtspVideoSourceError::InvalidAuthChallenge)?; - if let Some((_, algorithm)) = - params.iter().find(|(name, _)| name.eq_ignore_ascii_case("algorithm")) - { - if !algorithm.eq_ignore_ascii_case("MD5") { - return Err(RtspVideoSourceError::UnsupportedDigestAlgorithm(algorithm.clone())); + let mut builder = http_auth::PasswordClient::builder(); + for challenges in response.headers("www-authenticate") { + builder = builder.challenges(challenges); } + self.client = Some(builder.build().map_err(RtspVideoSourceError::Auth)?); + Ok(()) } - let qop = params - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("qop")) - .and_then(|(_, value)| select_digest_qop(value)); - let opaque = params - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("opaque")) - .map(|(_, value)| value.to_owned()); - - Ok(RtspAuthChallenge::Digest(DigestAuthChallenge { realm, nonce, opaque, qop })) } -fn strip_auth_scheme<'a>(header: &'a str, scheme: &str) -> Option<&'a str> { - let header = header.trim_start(); - let rest = header.get(scheme.len()..)?; - if !header[..scheme.len()].eq_ignore_ascii_case(scheme) { - return None; - } - if rest.is_empty() { - return Some(rest); - } - rest.strip_prefix(' ') -} +#[cfg(test)] +mod tests { + use super::*; -fn parse_auth_params(params: &str) -> Vec<(String, String)> { - let mut parsed = Vec::new(); - let mut current = String::new(); - let mut in_quotes = false; - let mut escaped = false; - for ch in params.chars() { - if escaped { - current.push(ch); - escaped = false; - continue; - } - match ch { - '\\' if in_quotes => { - escaped = true; - current.push(ch); - } - '"' => { - in_quotes = !in_quotes; - current.push(ch); - } - ',' if !in_quotes => { - push_auth_param(&mut parsed, ¤t); - current.clear(); - } - _ => current.push(ch), - } + fn context_with_credentials() -> RtspAuthContext { + RtspAuthContext::new(Some(RtspCredentials { + username: "user".to_owned(), + password: "pass".to_owned(), + })) } - push_auth_param(&mut parsed, ¤t); - parsed -} -fn push_auth_param(parsed: &mut Vec<(String, String)>, param: &str) { - let Some((name, value)) = param.trim().split_once('=') else { - return; - }; - parsed.push((name.trim().to_owned(), unquote_auth_value(value.trim()))); -} - -fn unquote_auth_value(value: &str) -> String { - let Some(value) = value.strip_prefix('"').and_then(|value| value.strip_suffix('"')) else { - return value.to_owned(); - }; - let mut unquoted = String::new(); - let mut escaped = false; - for ch in value.chars() { - if escaped { - unquoted.push(ch); - escaped = false; - } else if ch == '\\' { - escaped = true; - } else { - unquoted.push(ch); + fn unauthorized(challenges: &[&str]) -> RtspResponse { + let mut bytes = b"RTSP/1.0 401 Unauthorized\r\nCSeq: 1\r\n".to_vec(); + for challenge in challenges { + bytes.extend_from_slice(format!("WWW-Authenticate: {challenge}\r\n").as_bytes()); } + bytes.extend_from_slice(b"\r\n"); + RtspResponse::parse_for_tests(&bytes) } - unquoted -} -fn select_digest_qop(value: &str) -> Option { - value.split(',').map(str::trim).find(|qop| qop.eq_ignore_ascii_case("auth")).map(str::to_owned) -} - -fn build_digest_authorization( - credentials: &RtspCredentials, - challenge: &DigestAuthChallenge, - method: &str, - uri: &str, - nonce_count: u32, - cnonce: &str, -) -> String { - let ha1 = - md5_hex(format!("{}:{}:{}", credentials.username, challenge.realm, credentials.password)); - let ha2 = md5_hex(format!("{method}:{uri}")); - let response = if let Some(qop) = &challenge.qop { - md5_hex(format!("{ha1}:{}:{nonce_count:08x}:{cnonce}:{qop}:{ha2}", challenge.nonce)) - } else { - md5_hex(format!("{ha1}:{}:{ha2}", challenge.nonce)) - }; + #[test] + fn answers_basic_challenge() { + let mut context = context_with_credentials(); + context.update_from_unauthorized(&unauthorized(&["Basic realm=\"camera\""])).unwrap(); - let mut header = format!( - "Digest username=\"{}\", realm=\"{}\", nonce=\"{}\", uri=\"{}\", response=\"{}\"", - quote_auth_value(&credentials.username), - quote_auth_value(&challenge.realm), - quote_auth_value(&challenge.nonce), - quote_auth_value(uri), - response - ); - if let Some(qop) = &challenge.qop { - header.push_str(&format!( - ", qop={}, nc={nonce_count:08x}, cnonce=\"{}\"", - quote_auth_value(qop), - quote_auth_value(cnonce) - )); - } - if let Some(opaque) = &challenge.opaque { - header.push_str(&format!(", opaque=\"{}\"", quote_auth_value(opaque))); + let header = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + assert_eq!(header, "Basic dXNlcjpwYXNz"); } - header -} - -fn quote_auth_value(value: &str) -> String { - value.replace('\\', "\\\\").replace('"', "\\\"") -} - -fn md5_hex(input: impl AsRef<[u8]>) -> String { - format!("{:x}", Md5::digest(input)) -} - -/// Builds an unpredictable client nonce. Each [`RandomState`] draws fresh -/// OS-seeded keys, so chaining two independent states yields 128 bits of -/// entropy without adding an RNG dependency. -fn make_cnonce() -> String { - let mut hasher = RandomState::new().build_hasher(); - hasher.write_u64(0x6c6b_7274_7370); - let high = hasher.finish(); - let mut hasher = RandomState::new().build_hasher(); - hasher.write_u64(high); - let low = hasher.finish(); - format!("{high:016x}{low:016x}") -} - -#[cfg(test)] -mod tests { - use super::*; #[test] - fn builds_digest_authorization_with_qop_auth() { - // RFC 2617 section 3.5 example values. - let credentials = RtspCredentials { - username: "Mufasa".to_owned(), - password: "Circle Of Life".to_owned(), - }; - let challenge = DigestAuthChallenge { - realm: "testrealm@host.com".to_owned(), - nonce: "dcd98b7102dd2f0e8b11d0f600bfb0c093".to_owned(), - opaque: Some("5ccc069c403ebaf9f0171e9517f40e41".to_owned()), - qop: Some("auth".to_owned()), - }; - - let authorization = build_digest_authorization( - &credentials, - &challenge, - "GET", - "/dir/index.html", - 1, - "0a4f113b", - ); + fn prefers_digest_over_basic() { + let mut context = context_with_credentials(); + context + .update_from_unauthorized(&unauthorized(&[ + "Basic realm=\"camera\"", + "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"", + ])) + .unwrap(); - assert!(authorization.contains("response=\"6629fae49393a05397450978507c4ef1\"")); - assert!(authorization.contains("qop=auth")); - assert!(authorization.contains("nc=00000001")); - assert!(authorization.contains("opaque=\"5ccc069c403ebaf9f0171e9517f40e41\"")); + let header = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + assert!(header.starts_with("Digest "), "unexpected header: {header}"); + assert!(header.contains("username=\"user\"")); + assert!(header.contains("nc=00000001")); } #[test] - fn parses_digest_challenge_with_quoted_values() { - let challenge = parse_authenticate_header(&[ - "Digest realm=\"a, \\\"quoted\\\" realm\", nonce=\"abc\", qop=\"auth,auth-int\"", - ]) - .unwrap(); + fn digest_nonce_count_increments_across_requests() { + let mut context = context_with_credentials(); + context + .update_from_unauthorized(&unauthorized(&[ + "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"", + ])) + .unwrap(); - assert_eq!( - challenge, - RtspAuthChallenge::Digest(DigestAuthChallenge { - realm: "a, \"quoted\" realm".to_owned(), - nonce: "abc".to_owned(), - opaque: None, - qop: Some("auth".to_owned()), - }) - ); + let _ = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + let second = context.header("SETUP", "rtsp://camera.example/live").unwrap().unwrap(); + assert!(second.contains("nc=00000002"), "unexpected header: {second}"); } #[test] - fn prefers_digest_over_basic() { - let challenge = parse_authenticate_header(&[ - "Basic realm=\"camera\"", - "Digest realm=\"camera\", nonce=\"abc\"", - ]) - .unwrap(); - assert!(matches!(challenge, RtspAuthChallenge::Digest(_))); + fn rejects_unsupported_scheme() { + let mut context = context_with_credentials(); + let err = + context.update_from_unauthorized(&unauthorized(&["Bearer token=\"abc\""])).unwrap_err(); + assert!(matches!(err, RtspVideoSourceError::Auth(_)), "unexpected error: {err:?}"); } #[test] - fn rejects_unsupported_auth_scheme() { - let err = parse_authenticate_header(&["Bearer token=\"abc\""]).unwrap_err(); - match err { - RtspVideoSourceError::UnsupportedAuthScheme(scheme) => assert_eq!(scheme, "Bearer"), - other => panic!("expected unsupported auth scheme, got {other:?}"), - } + fn requires_credentials_for_challenges() { + let mut context = RtspAuthContext::new(None); + let err = + context.update_from_unauthorized(&unauthorized(&["Basic realm=\"c\""])).unwrap_err(); + assert!(matches!(err, RtspVideoSourceError::MissingCredentials)); } #[test] - fn rejects_unsupported_digest_algorithm() { - let err = parse_authenticate_header(&[ - "Digest realm=\"camera\", nonce=\"abc\", algorithm=SHA-256", - ]) - .unwrap_err(); - assert!(matches!(err, RtspVideoSourceError::UnsupportedDigestAlgorithm(algorithm) if algorithm == "SHA-256")); + fn no_header_before_challenge() { + let mut context = context_with_credentials(); + assert!(context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().is_none()); } #[test] @@ -393,9 +181,4 @@ mod tests { assert!(debug.contains("admin")); assert!(!debug.contains("secret")); } - - #[test] - fn cnonces_are_distinct() { - assert_ne!(make_cnonce(), make_cnonce()); - } } diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index a61b223bc..c483f7f34 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -709,6 +709,14 @@ fn find_header_end(buf: &[u8]) -> Option { buf.windows(4).take(MAX_HEADER_BYTES).position(|window| window == b"\r\n\r\n") } +#[cfg(test)] +impl RtspResponse { + /// Parses one complete response, for tests in sibling modules. + pub(super) fn parse_for_tests(bytes: &[u8]) -> Self { + parse_response(bytes).expect("invalid response").expect("incomplete response").0 + } +} + /// Extracts the session identifier from a `Session` header value. pub(super) fn parse_session_id(session_header: &str) -> Result { let session_id = session_header.split(';').next().unwrap_or_default().trim(); diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index 4595e3863..d77eacb9a 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -190,15 +190,10 @@ pub enum RtspVideoSourceError { /// The server requires authentication but no credentials were supplied. #[error("RTSP authentication required but no credentials were supplied")] MissingCredentials, - /// The authentication challenge was malformed. - #[error("invalid RTSP authentication challenge")] - InvalidAuthChallenge, - /// The authentication scheme is not supported. - #[error("unsupported RTSP authentication scheme: {0}")] - UnsupportedAuthScheme(String), - /// The Digest algorithm is not supported. - #[error("unsupported RTSP Digest algorithm: {0}")] - UnsupportedDigestAlgorithm(String), + /// The authentication challenge was malformed or unsupported, or the + /// response to it could not be built. + #[error("RTSP authentication failed: {0}")] + Auth(String), /// The SDP was missing a supported video track. #[error("RTSP SDP does not contain a supported video track")] MissingVideoTrack, From 3a2854d6fcf49f1bb3042e14f72490de6a236666 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:25:50 -0700 Subject: [PATCH 21/38] Use rtsp-types crate --- Cargo.lock | 22 ++ livekit-capture/Cargo.toml | 3 +- livekit-capture/src/sources/rtsp/client.rs | 285 +++++++++++---------- 3 files changed, 174 insertions(+), 136 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3e3f39096..f9e9405f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1162,6 +1162,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -3988,6 +3997,7 @@ dependencies = [ "log", "pollster", "rcgen", + "rtsp-types", "rustls", "rustls-native-certs", "rustls-pki-types", @@ -6558,6 +6568,18 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153" +[[package]] +name = "rtsp-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47bf238c3da7994ef66a71724f385cb1ee25dcf04a156402d2727ad84dc1433" +dependencies = [ + "cookie-factory", + "nom 7.1.3", + "tinyvec", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.28" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index f4a297990..78a84dd0b 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -19,6 +19,7 @@ gio = { version = "0.22", optional = true } gstreamer-rtsp = { version = "0.25", optional = true } gstreamer-rtsp-server = { version = "0.25", optional = true } http-auth = { version = "0.1", optional = true } +rtsp-types = { version = "0.1", optional = true } rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"], optional = true } rustls-native-certs = { version = "0.8", optional = true } rustls-pki-types = { version = "1", optional = true } @@ -49,7 +50,7 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] -source-rtsp = ["dep:base64", "dep:http-auth"] +source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types"] # Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. # Certificates are verified against the system roots by default. source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index c483f7f34..c581cb90e 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -24,6 +24,7 @@ use std::{ }; use bytes::{Buf, Bytes, BytesMut}; +use rtsp_types::{headers, HeaderName, Message, Method, ParseError, Url, Version}; use super::{auth::RtspAuthContext, auth::RtspCredentials, RtspPhase, RtspVideoSourceError}; @@ -35,8 +36,10 @@ const READ_POLL: Duration = Duration::from_millis(100); /// connection is gone. const WRITE_TIMEOUT: Duration = Duration::from_secs(5); -/// Upper bound on an RTSP response header. -const MAX_HEADER_BYTES: usize = 64 * 1024; +/// Upper bound on one buffered-but-incomplete RTSP message. Interleaved +/// frames are at most 4 + 65535 bytes, and responses are far smaller, so an +/// incomplete message larger than this means the framing is corrupt. +const MAX_PENDING_MESSAGE_BYTES: usize = 128 * 1024; /// Bytes requested from the socket per read. const READ_CHUNK_BYTES: usize = 8 * 1024; @@ -47,8 +50,6 @@ pub(super) struct RtspUrl { /// Request URI with any userinfo stripped, so credentials never appear /// on the wire outside the `Authorization` header. pub(super) request_uri: String, - /// Value for the `Host` header, always including the port. - pub(super) host_header: String, /// Credentials from the URL userinfo, percent-decoded. pub(super) credentials: Option, /// Whether the URL requires TLS (`rtsps://`). @@ -81,15 +82,9 @@ impl RtspUrl { } let default_port = if tls { 322 } else { 554 }; let (connect_host, port) = parse_host_port(host_port, default_port)?; - let host_header = if host_port.contains(':') { - host_port.to_owned() - } else { - format!("{host_port}:{port}") - }; Ok(Self { request_uri: format!("{scheme}{host_port}{path_suffix}"), - host_header, credentials, tls, connect_host, @@ -166,6 +161,19 @@ pub(super) struct RtspResponse { } impl RtspResponse { + /// Converts a parsed `rtsp-types` response into the client's view. + fn from_message(response: rtsp_types::Response>) -> Self { + Self { + status_code: response.status().into(), + reason: response.reason_phrase().to_owned(), + headers: response + .headers() + .map(|(name, value)| (name.as_str().to_owned(), value.as_str().to_owned())) + .collect(), + body: response.into_body(), + } + } + pub(super) fn is_success(&self) -> bool { (200..300).contains(&self.status_code) } @@ -266,8 +274,8 @@ pub(super) struct RtspClient { scratch: Vec, cseq: u32, auth: RtspAuthContext, - host_header: String, last_read_at: Instant, + logged_server_request: bool, } // Manual so the read buffer's contents are not dumped; the authentication @@ -275,7 +283,6 @@ pub(super) struct RtspClient { impl fmt::Debug for RtspClient { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("RtspClient") - .field("host_header", &self.host_header) .field("cseq", &self.cseq) .finish_non_exhaustive() } @@ -340,8 +347,8 @@ impl RtspClient { scratch: vec![0; READ_CHUNK_BYTES], cseq: 1, auth: RtspAuthContext::new(credentials), - host_header: url.host_header.clone(), last_read_at: Instant::now(), + logged_server_request: false, }) } @@ -380,29 +387,33 @@ impl RtspClient { &mut self, method: &str, uri: &str, - headers: &[(&str, &str)], + extra_headers: &[(&str, &str)], ) -> Result<(), RtspVideoSourceError> { - use fmt::Write as _; - let cseq = self.cseq; self.cseq = self.cseq.saturating_add(1); let authorization = self.auth.header(method, uri)?; + let request_uri = Url::parse(uri) + .map_err(|_| RtspVideoSourceError::InvalidUrl("request URI is not a valid URL"))?; - let mut request = String::with_capacity(256); - // Writing to a `String` cannot fail. - let _ = write!(request, "{method} {uri} RTSP/1.0\r\n"); - let _ = write!(request, "CSeq: {cseq}\r\n"); - let _ = write!(request, "User-Agent: livekit-capture/0.1\r\n"); - let _ = write!(request, "Host: {}\r\n", self.host_header); + let mut request = rtsp_types::Request::builder(request_method(method), Version::V1_0) + .request_uri(request_uri) + .header(headers::CSEQ, cseq.to_string()) + .header(headers::USER_AGENT, "livekit-capture/0.1".to_owned()); if let Some(authorization) = authorization { - let _ = write!(request, "Authorization: {authorization}\r\n"); + request = request.header(headers::AUTHORIZATION, authorization); } - for (name, value) in headers { - let _ = write!(request, "{name}: {value}\r\n"); + for (name, value) in extra_headers { + // Header names come from this crate's own call sites. + let name = HeaderName::try_from(*name).expect("static header names are valid"); + request = request.header(name, (*value).to_owned()); } - request.push_str("\r\n"); - self.stream.write_all(request.as_bytes())?; + let mut bytes = Vec::with_capacity(256); + request + .empty() + .write(&mut bytes) + .map_err(|err| RtspVideoSourceError::Io(io::Error::other(err)))?; + self.stream.write_all(&bytes)?; self.stream.flush()?; Ok(()) } @@ -414,9 +425,26 @@ impl RtspClient { phase: RtspPhase, ) -> Result { loop { - if let Some((response, consumed)) = parse_response(&self.buf)? { - self.buf.advance(consumed); - return Ok(response); + if !self.buf.is_empty() { + match Message::parse(&self.buf) { + Ok((message, consumed)) => { + self.buf.advance(consumed); + match message { + Message::Response(response) => { + return Ok(RtspResponse::from_message(response)); + } + Message::Data(_) | Message::Request(_) => { + return Err(RtspVideoSourceError::InvalidResponse( + "expected a response", + )); + } + } + } + Err(ParseError::Incomplete(_)) => self.check_pending_size()?, + Err(ParseError::Error) => { + return Err(RtspVideoSourceError::InvalidResponse("malformed response")); + } + } } match self.fill()? { StreamFill::Filled => {} @@ -432,6 +460,14 @@ impl RtspClient { } } + /// Fails when a still-incomplete message exceeds the framing bound. + fn check_pending_size(&self) -> Result<(), RtspVideoSourceError> { + if self.buf.len() > MAX_PENDING_MESSAGE_BYTES { + return Err(RtspVideoSourceError::InvalidResponse("message too large")); + } + Ok(()) + } + /// Reads the next interleaved unit, returning within roughly one /// [`READ_POLL`] when the stream is silent. Framing state survives /// timed-out reads. @@ -461,31 +497,43 @@ impl RtspClient { /// Parses one complete unit from the front of the buffer. fn parse_front(&mut self) -> Result, RtspVideoSourceError> { - let Some(&magic) = self.buf.first() else { - return Ok(None); - }; - match magic { - b'$' => { - if self.buf.len() < 4 { - return Ok(None); + loop { + if self.buf.is_empty() { + return Ok(None); + } + match Message::parse(&self.buf) { + Ok((message, consumed)) => { + self.buf.advance(consumed); + match message { + Message::Data(data) => { + let channel = data.channel_id(); + let payload = Bytes::from(data.into_body()); + return Ok(Some(InterleavedPoll::Frame { channel, payload })); + } + Message::Response(response) => { + return Ok(Some(InterleavedPoll::Response( + RtspResponse::from_message(response), + ))); + } + Message::Request(request) => { + // Some servers send requests (ANNOUNCE, keepalive + // checks) to the client mid-stream; ignore them. + if !self.logged_server_request { + self.logged_server_request = true; + log::warn!( + "ignoring in-band RTSP {:?} request from the server", + request.method(), + ); + } + } + } } - let channel = self.buf[1]; - let payload_len = u16::from_be_bytes([self.buf[2], self.buf[3]]) as usize; - if self.buf.len() < 4 + payload_len { + Err(ParseError::Incomplete(_)) => { + self.check_pending_size()?; return Ok(None); } - self.buf.advance(4); - let payload = self.buf.split_to(payload_len).freeze(); - Ok(Some(InterleavedPoll::Frame { channel, payload })) + Err(ParseError::Error) => return Err(RtspVideoSourceError::UnexpectedData), } - b'R' => match parse_response(&self.buf)? { - Some((response, consumed)) => { - self.buf.advance(consumed); - Ok(Some(InterleavedPoll::Response(response))) - } - None => Ok(None), - }, - _ => Err(RtspVideoSourceError::UnexpectedData), } } @@ -651,69 +699,26 @@ fn is_timeout_io_error(err: &io::Error) -> bool { matches!(err.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) } -/// Parses one RTSP response from the front of `buf`, returning the response -/// and the bytes it consumed, or `Ok(None)` when more bytes are needed. -fn parse_response(buf: &[u8]) -> Result, RtspVideoSourceError> { - let Some(header_end) = find_header_end(buf) else { - if buf.len() > MAX_HEADER_BYTES { - return Err(RtspVideoSourceError::InvalidResponse("header too large")); - } - return Ok(None); - }; - - let header_text = str::from_utf8(&buf[..header_end]) - .map_err(|_| RtspVideoSourceError::InvalidResponse("header is not UTF-8"))?; - let mut lines = header_text.split("\r\n"); - let status_line = - lines.next().ok_or(RtspVideoSourceError::InvalidResponse("missing status line"))?; - let mut status_parts = status_line.splitn(3, ' '); - if status_parts.next() != Some("RTSP/1.0") { - return Err(RtspVideoSourceError::InvalidResponse("unsupported version")); - } - let status_code = status_parts - .next() - .ok_or(RtspVideoSourceError::InvalidResponse("missing status code"))? - .parse() - .map_err(|_| RtspVideoSourceError::InvalidResponse("invalid status code"))?; - let reason = status_parts.next().unwrap_or_default().to_owned(); - - let mut headers = Vec::new(); - for line in lines { - let Some((name, value)) = line.split_once(':') else { - return Err(RtspVideoSourceError::InvalidResponse("malformed header")); - }; - headers.push((name.trim().to_owned(), value.trim().to_owned())); - } - - let content_length = headers - .iter() - .find(|(name, _)| name.eq_ignore_ascii_case("content-length")) - .map(|(_, value)| value.parse::()) - .transpose() - .map_err(|_| RtspVideoSourceError::InvalidResponse("invalid content length"))? - .unwrap_or(0); - let body_start = header_end + 4; - let Some(consumed) = body_start.checked_add(content_length) else { - return Err(RtspVideoSourceError::InvalidResponse("invalid content length")); - }; - if buf.len() < consumed { - return Ok(None); - } - let body = buf[body_start..consumed].to_vec(); - - Ok(Some((RtspResponse { status_code, reason, headers, body }, consumed))) -} - -/// Finds the end of the response header (the start of `\r\n\r\n`). -fn find_header_end(buf: &[u8]) -> Option { - buf.windows(4).take(MAX_HEADER_BYTES).position(|window| window == b"\r\n\r\n") +/// Maps a request method from this crate's own call sites to its typed form. +fn request_method(method: &str) -> Method { + match method { + "DESCRIBE" => Method::Describe, + "SETUP" => Method::Setup, + "PLAY" => Method::Play, + "OPTIONS" => Method::Options, + "TEARDOWN" => Method::Teardown, + other => Method::Extension(other.to_owned()), + } } #[cfg(test)] impl RtspResponse { /// Parses one complete response, for tests in sibling modules. pub(super) fn parse_for_tests(bytes: &[u8]) -> Self { - parse_response(bytes).expect("invalid response").expect("incomplete response").0 + match Message::parse(bytes).expect("invalid response") { + (Message::Response(response), _) => Self::from_message(response), + (other, _) => panic!("expected a response, got {other:?}"), + } } } @@ -764,7 +769,6 @@ mod tests { let url = RtspUrl::parse("rtsp://admin:secret@camera.example:554/live").unwrap(); assert_eq!(url.request_uri, "rtsp://camera.example:554/live"); - assert_eq!(url.host_header, "camera.example:554"); assert_eq!( url.credentials, Some(RtspCredentials { username: "admin".to_owned(), password: "secret".to_owned() }) @@ -788,7 +792,6 @@ mod tests { let url = RtspUrl::parse("rtsps://camera.example/live").unwrap(); assert!(url.tls); assert_eq!(url.port, 322); - assert_eq!(url.host_header, "camera.example:322"); assert_eq!(url.request_uri, "rtsps://camera.example/live"); let url = RtspUrl::parse("rtsps://admin:secret@camera.example:7441/live").unwrap(); @@ -804,7 +807,6 @@ mod tests { fn defaults_to_port_554() { let url = RtspUrl::parse("rtsp://camera.example/live").unwrap(); assert_eq!(url.port, 554); - assert_eq!(url.host_header, "camera.example:554"); assert_eq!(url.request_uri, "rtsp://camera.example/live"); } @@ -813,7 +815,6 @@ mod tests { let url = RtspUrl::parse("rtsp://[2001:db8::1]:8554/live").unwrap(); assert_eq!(url.connect_host, "2001:db8::1"); assert_eq!(url.port, 8554); - assert_eq!(url.host_header, "[2001:db8::1]:8554"); assert_eq!(url.request_uri, "rtsp://[2001:db8::1]:8554/live"); } @@ -838,34 +839,48 @@ mod tests { } #[test] - fn parses_response_with_body_and_reports_consumed_bytes() { - let bytes = - b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 4\r\n\r\nbody$leftover"; - let (response, consumed) = parse_response(bytes).unwrap().unwrap(); + fn rejects_oversized_incomplete_message() { + use std::{net::TcpListener, thread}; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + // Claim a body far larger than the framing bound, stream past + // the bound without completing it, then stall. + stream + .write_all(b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 999999\r\n\r\n") + .unwrap(); + stream.write_all(&vec![b'a'; MAX_PENDING_MESSAGE_BYTES + 8 * 1024]).unwrap(); + stream.flush().unwrap(); + thread::sleep(Duration::from_millis(200)); + }); + + let url = RtspUrl::parse(&format!("rtsp://{addr}/test")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + let mut client = RtspClient::connect(&url, None, false, deadline).unwrap(); + let err = client + .request("DESCRIBE", &url.request_uri, &[], deadline, RtspPhase::Describe) + .unwrap_err(); + + assert!( + matches!(err, RtspVideoSourceError::InvalidResponse("message too large")), + "unexpected error: {err:?}" + ); + server.join().unwrap(); + } + + #[test] + fn converts_parsed_responses() { + let response = RtspResponse::parse_for_tests( + b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 4\r\n\r\nbody", + ); assert_eq!(response.status_code, 200); assert_eq!(response.reason, "OK"); assert_eq!(response.header("cseq"), Some("1")); assert_eq!(response.body, b"body"); - assert_eq!(&bytes[consumed..], b"$leftover"); - } - - #[test] - fn incomplete_response_needs_more_bytes() { - assert!(parse_response(b"RTSP/1.0 200 OK\r\nCSeq:").unwrap().is_none()); - assert!(parse_response(b"RTSP/1.0 200 OK\r\nContent-Length: 4\r\n\r\nbo") - .unwrap() - .is_none()); - } - - #[test] - fn rejects_oversized_header() { - let mut bytes = b"RTSP/1.0 200 OK\r\n".to_vec(); - bytes.resize(MAX_HEADER_BYTES + 8, b'a'); - assert!(matches!( - parse_response(&bytes), - Err(RtspVideoSourceError::InvalidResponse("header too large")) - )); + assert!(response.is_success()); } #[test] From d4996b69170867362d41e8133d6236a2979232d1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:28:54 -0700 Subject: [PATCH 22/38] Use sdp-types crate --- Cargo.lock | 28 +++++ livekit-capture/Cargo.toml | 3 +- livekit-capture/src/sources/rtsp/mod.rs | 4 +- livekit-capture/src/sources/rtsp/sdp.rs | 154 ++++++++++-------------- 4 files changed, 92 insertions(+), 97 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f9e9405f0..7418a2584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,6 +735,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", +] + [[package]] name = "built" version = "0.8.1" @@ -2065,6 +2074,12 @@ dependencies = [ "rand 0.9.5", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "2.5.0" @@ -4002,6 +4017,7 @@ dependencies = [ "rustls-native-certs", "rustls-pki-types", "schemars", + "sdp-types", "serde", "test-log", "thiserror 2.0.19", @@ -6842,6 +6858,18 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "sdp-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46d46be065d0565cf5c6703b6ba9f92b0de85b20a4c83f87e110bc0a1040b1bc" +dependencies = [ + "bstr", + "fallible-iterator", + "hex", + "thiserror 2.0.19", +] + [[package]] name = "security-framework" version = "3.7.0" diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 78a84dd0b..e19e7a11f 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -23,6 +23,7 @@ rtsp-types = { version = "0.1", optional = true } rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"], optional = true } rustls-native-certs = { version = "0.8", optional = true } rustls-pki-types = { version = "1", optional = true } +sdp-types = { version = "0.2", optional = true } livekit = { workspace = true } log = { workspace = true } pollster = { version = "0.4", optional = true } @@ -50,7 +51,7 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] -source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types"] +source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types", "dep:sdp-types"] # Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. # Certificates are verified against the system roots by default. source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index d77eacb9a..6f1cd20f9 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -336,15 +336,13 @@ impl RtspVideoSource { handshake_deadline, RtspPhase::Describe, )?; - let sdp_text = - str::from_utf8(&describe.body).map_err(|_| RtspVideoSourceError::InvalidSdp)?; // Relative control URLs resolve against the Content-Base per // RFC 2326 appendix C.1.1, falling back to the request URL. let base_url = describe .header("content-base") .or_else(|| describe.header("content-location")) .unwrap_or(&url.request_uri); - let session = sdp::parse_sdp_session(base_url, sdp_text, config.codec)?; + let session = sdp::parse_sdp_session(base_url, &describe.body, config.codec)?; let setup = client.request( "SETUP", diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs index 97d5760cb..7b43a5ad9 100644 --- a/livekit-capture/src/sources/rtsp/sdp.rs +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -49,15 +49,6 @@ pub(super) struct SdpVideoTrack { pub(super) framesize: Option, } -#[derive(Debug, Clone, Default)] -struct PartialVideoTrack { - payload_types: Vec, - rtp_maps: Vec, - fmtps: Vec<(u8, String)>, - framesizes: Vec<(u8, VideoResolution)>, - control: Option, -} - #[derive(Debug, Clone, PartialEq, Eq)] struct SdpRtpMap { payload_type: u8, @@ -73,62 +64,26 @@ struct SdpRtpMap { /// one, the first supported video track is selected. pub(super) fn parse_sdp_session( base_url: &str, - sdp: &str, + sdp: &[u8], expected_codec: Option, ) -> Result { - let mut session_control = None; - let mut tracks = Vec::new(); - let mut current: Option = None; - let mut in_media_section = false; - - for line in sdp.lines().map(str::trim).filter(|line| !line.is_empty()) { - if let Some(media) = line.strip_prefix("m=") { - if let Some(track) = current.take() { - tracks.push(track); - } - in_media_section = true; - if let Some(video) = media.strip_prefix("video ") { - current = Some(parse_video_media(video)); - } - continue; - } + let session = sdp_types::Session::parse(sdp).map_err(|err| { + log::debug!("failed to parse SDP: {err}"); + RtspVideoSourceError::InvalidSdp + })?; + let session_control = attribute_value(&session.attributes, "control"); - if !in_media_section { - if let Some(control) = line.strip_prefix("a=control:") { - session_control = Some(control.trim().to_owned()); - } - continue; - } - - let Some(track) = current.as_mut() else { + let mut offered = Vec::new(); + for media in &session.medias { + if media.media != "video" { continue; - }; - if let Some(control) = line.strip_prefix("a=control:") { - track.control = Some(control.trim().to_owned()); - } else if let Some(rtpmap) = line.strip_prefix("a=rtpmap:") { - if let Some(rtp_map) = parse_rtpmap(rtpmap) { - track.rtp_maps.push(rtp_map); - } - } else if let Some(fmtp) = line.strip_prefix("a=fmtp:") { - if let Some((payload_type, params)) = fmtp.trim().split_once(char::is_whitespace) { - if let Ok(payload_type) = payload_type.parse() { - track.fmtps.push((payload_type, params.to_owned())); - } - } - } else if let Some(framesize) = line.strip_prefix("a=framesize:") { - if let Some(parsed) = parse_framesize(framesize) { - track.framesizes.push(parsed); - } } - } - if let Some(track) = current { - tracks.push(track); - } + let rtp_maps: Vec = attribute_values(&media.attributes, "rtpmap") + .filter_map(parse_rtpmap) + .collect(); - let mut offered = Vec::new(); - for track in tracks { - for payload_type in &track.payload_types { - let Some(rtp_map) = track.rtp_maps.iter().find(|map| map.payload_type == *payload_type) + for payload_type in media.fmt.split_whitespace().filter_map(|pt| pt.parse::().ok()) { + let Some(rtp_map) = rtp_maps.iter().find(|map| map.payload_type == payload_type) else { continue; }; @@ -141,28 +96,33 @@ pub(super) fn parse_sdp_session( } } - let parameter_sets = track - .fmtps - .iter() - .find(|(fmtp_payload_type, _)| fmtp_payload_type == payload_type) - .map(|(_, params)| parse_fmtp_parameter_sets(rtp_map.codec, params)) + let parameter_sets = attribute_values(&media.attributes, "fmtp") + .find_map(|value| { + let (fmtp_payload_type, params) = + value.trim().split_once(char::is_whitespace)?; + (fmtp_payload_type.parse::().ok()? == payload_type) + .then(|| parse_fmtp_parameter_sets(rtp_map.codec, params)) + }) .unwrap_or_default(); - let framesize = track - .framesizes - .iter() - .find(|(framesize_payload_type, _)| framesize_payload_type == payload_type) - .map(|(_, resolution)| *resolution); + let framesize = attribute_values(&media.attributes, "framesize") + .filter_map(parse_framesize) + .find_map(|(framesize_payload_type, resolution)| { + (framesize_payload_type == payload_type).then_some(resolution) + }); return Ok(SdpSession { video: SdpVideoTrack { codec: rtp_map.codec, - payload_type: *payload_type, + payload_type, clock_rate: rtp_map.clock_rate, - control_url: resolve_control_url(base_url, track.control.as_deref()), + control_url: resolve_control_url( + base_url, + attribute_value(&media.attributes, "control"), + ), parameter_sets, framesize, }, - aggregate_control_url: resolve_control_url(base_url, session_control.as_deref()), + aggregate_control_url: resolve_control_url(base_url, session_control), }); } } @@ -175,13 +135,21 @@ pub(super) fn parse_sdp_session( } } -fn parse_video_media(media: &str) -> PartialVideoTrack { - let payload_types = media - .split_whitespace() - .skip(2) - .filter_map(|payload_type| payload_type.parse().ok()) - .collect(); - PartialVideoTrack { payload_types, ..Default::default() } +/// Returns the first value of the named attribute. +fn attribute_value<'a>(attributes: &'a [sdp_types::Attribute], name: &'a str) -> Option<&'a str> { + attribute_values(attributes, name).next() +} + +/// Returns every value of the named attribute. +fn attribute_values<'a>( + attributes: &'a [sdp_types::Attribute], + name: &'a str, +) -> impl Iterator + 'a { + attributes + .iter() + .filter(move |attribute| attribute.attribute == name) + .filter_map(|attribute| attribute.value.as_deref()) + .map(str::trim) } fn parse_rtpmap(rtpmap: &str) -> Option { @@ -313,7 +281,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=1\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); assert_eq!(session.video.codec, EncodedVideoCodec::H264); assert_eq!(session.video.payload_type, 96); @@ -339,7 +307,7 @@ a=control:trackID=1\r\n\ a=rtpmap:96 {rtpmap}\r\n" ); - let session = parse_sdp_session(BASE_URL, &sdp, Some(codec)).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(codec)).unwrap(); assert_eq!(session.video.codec, codec); assert_eq!(session.video.payload_type, 96); @@ -355,7 +323,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=1\r\n\ a=rtpmap:96 VP9/90000\r\n"; - let err = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::AV1)).unwrap_err(); + let err = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::AV1)).unwrap_err(); match err { RtspVideoSourceError::CodecMismatch { expected, offered } => { @@ -375,7 +343,7 @@ a=control:trackID=1\r\n\ a=rtpmap:98 H265/90000\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); assert_eq!(session.video.codec, EncodedVideoCodec::H264); assert_eq!(session.video.payload_type, 96); @@ -392,7 +360,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=2\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::H264)).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); assert_eq!(session.video.codec, EncodedVideoCodec::H264); assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); @@ -407,7 +375,7 @@ a=control:trackID=1\r\n\ a=rtpmap:98 H265/90000\r\n\ a=rtpmap:96 H264/90000\r\n"; - let err = parse_sdp_session(BASE_URL, sdp, Some(EncodedVideoCodec::VP8)).unwrap_err(); + let err = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::VP8)).unwrap_err(); match err { RtspVideoSourceError::CodecMismatch { expected, offered } => { @@ -429,7 +397,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=2\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.codec, EncodedVideoCodec::H264); assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); @@ -444,7 +412,7 @@ m=video 0 RTP/AVP 96\r\n\ a=rtpmap:96 H264/90000\r\n\ a=fmtp:96 packetization-mode=1;sprop-parameter-sets=ZwlA,aAlB;profile-level-id=42e01e\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); assert_eq!(session.video.parameter_sets.pps, vec![vec![0x68, 0x09, 0x41]]); @@ -460,7 +428,7 @@ m=video 0 RTP/AVP 96\r\n\ a=rtpmap:96 H265/90000\r\n\ a=fmtp:96 sprop-vps=QAEB;sprop-sps=QgEC;sprop-pps=RAED\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.parameter_sets.vps, vec![vec![0x40, 0x01, 0x01]]); assert_eq!(session.video.parameter_sets.sps, vec![vec![0x42, 0x01, 0x02]]); @@ -475,7 +443,7 @@ m=video 0 RTP/AVP 96\r\n\ a=rtpmap:96 H264/90000\r\n\ a=fmtp:96 sprop-parameter-sets=!!!not-base64!!!,ZwlA\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); assert!(session.video.parameter_sets.pps.is_empty()); @@ -489,7 +457,7 @@ m=video 0 RTP/AVP 96\r\n\ a=rtpmap:96 H264/90000\r\n\ a=framesize:96 1280-720\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.framesize, Some(VideoResolution::new(1280, 720))); } @@ -503,7 +471,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=1\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.aggregate_control_url, "rtsp://camera.example/live/aggregate"); assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=1"); @@ -518,7 +486,7 @@ m=video 0 RTP/AVP 96\r\n\ a=control:trackID=1\r\n\ a=rtpmap:96 H264/90000\r\n"; - let session = parse_sdp_session(BASE_URL, sdp, None).unwrap(); + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); assert_eq!(session.aggregate_control_url, BASE_URL); } @@ -559,7 +527,7 @@ a=control:trackID=1\r\n\ a=rtpmap:96 H264/90000\r\n"; let session = - parse_sdp_session("rtsp://camera.example/relocated/", sdp, None).unwrap(); + parse_sdp_session("rtsp://camera.example/relocated/", sdp.as_bytes(), None).unwrap(); assert_eq!(session.video.control_url, "rtsp://camera.example/relocated/trackID=1"); } From aafe9d4c277699b06983d079274372676df72604 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:30:56 -0700 Subject: [PATCH 23/38] Use url crate --- Cargo.lock | 1 + livekit-capture/Cargo.toml | 3 +- livekit-capture/src/sources/rtsp/client.rs | 92 ++++++++-------------- 3 files changed, 36 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7418a2584..9b343c144 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4022,6 +4022,7 @@ dependencies = [ "test-log", "thiserror 2.0.19", "tokio", + "url", "wgpu", "yuv-sys", ] diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index e19e7a11f..8c2b1c3ab 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -24,6 +24,7 @@ rustls = { version = "0.23", default-features = false, features = ["ring", "logg rustls-native-certs = { version = "0.8", optional = true } rustls-pki-types = { version = "1", optional = true } sdp-types = { version = "0.2", optional = true } +url = { version = "2", optional = true } livekit = { workspace = true } log = { workspace = true } pollster = { version = "0.4", optional = true } @@ -51,7 +52,7 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] -source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types", "dep:sdp-types"] +source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types", "dep:sdp-types", "dep:url"] # Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. # Certificates are verified against the system roots by default. source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index c581cb90e..b77949ffa 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -61,47 +61,46 @@ pub(super) struct RtspUrl { impl RtspUrl { /// Parses an `rtsp(s)://[user:password@]host[:port][/path]` URL. pub(super) fn parse(url: &str) -> Result { - let (scheme, rest, tls) = if let Some(rest) = url.strip_prefix("rtsp://") { - ("rtsp://", rest, false) - } else if let Some(rest) = url.strip_prefix("rtsps://") { - ("rtsps://", rest, true) - } else { - return Err(RtspVideoSourceError::InvalidUrl("expected rtsp:// or rtsps:// scheme")); + let parsed = + Url::parse(url).map_err(|_| RtspVideoSourceError::InvalidUrl("malformed URL"))?; + let tls = match parsed.scheme() { + "rtsp" => false, + "rtsps" => true, + _ => { + return Err(RtspVideoSourceError::InvalidUrl( + "expected rtsp:// or rtsps:// scheme", + )) + } }; - let (authority, path_suffix) = match rest.find('/') { - Some(path_start) => (&rest[..path_start], &rest[path_start..]), - None => (rest, ""), + + // `Host::Ipv6` renders unbracketed, as `ToSocketAddrs` expects. + let connect_host = match parsed.host() { + Some(url::Host::Domain(domain)) if !domain.is_empty() => domain.to_owned(), + Some(url::Host::Ipv4(address)) => address.to_string(), + Some(url::Host::Ipv6(address)) => address.to_string(), + _ => return Err(RtspVideoSourceError::InvalidUrl("missing host")), }; + let port = parsed.port().unwrap_or(if tls { 322 } else { 554 }); - let (credentials, host_port) = match authority.rsplit_once('@') { - Some((userinfo, host_port)) => (Some(parse_userinfo(userinfo)?), host_port), - None => (None, authority), + let credentials = if parsed.username().is_empty() && parsed.password().is_none() { + None + } else { + let username = percent_decode(parsed.username()); + if username.is_empty() { + return Err(RtspVideoSourceError::InvalidUrl("missing username")); + } + let password = parsed.password().map(percent_decode).unwrap_or_default(); + Some(RtspCredentials { username, password }) }; - if host_port.is_empty() { - return Err(RtspVideoSourceError::InvalidUrl("missing host")); - } - let default_port = if tls { 322 } else { 554 }; - let (connect_host, port) = parse_host_port(host_port, default_port)?; - Ok(Self { - request_uri: format!("{scheme}{host_port}{path_suffix}"), - credentials, - tls, - connect_host, - port, - }) - } -} + // Strip userinfo so credentials never appear on the wire outside + // the `Authorization` header. + let mut request_uri = parsed; + let _ = request_uri.set_username(""); + let _ = request_uri.set_password(None); -fn parse_userinfo(userinfo: &str) -> Result { - let (username, password) = userinfo.split_once(':').unwrap_or((userinfo, "")); - if username.is_empty() { - return Err(RtspVideoSourceError::InvalidUrl("missing username")); + Ok(Self { request_uri: request_uri.to_string(), credentials, tls, connect_host, port }) } - Ok(RtspCredentials { - username: percent_decode(username), - password: percent_decode(password), - }) } /// Decodes RFC 3986 percent-escapes; malformed escapes pass through as-is. @@ -126,31 +125,6 @@ fn percent_decode(value: &str) -> String { String::from_utf8_lossy(&decoded).into_owned() } -fn parse_host_port( - host_port: &str, - default_port: u16, -) -> Result<(String, u16), RtspVideoSourceError> { - if let Some(rest) = host_port.strip_prefix('[') { - let Some((host, after_host)) = rest.split_once(']') else { - return Err(RtspVideoSourceError::InvalidUrl("malformed IPv6 host")); - }; - let port = after_host.strip_prefix(':').map(parse_port).transpose()?.unwrap_or(default_port); - return Ok((host.to_owned(), port)); - } - - if let Some((host, port)) = host_port.rsplit_once(':') { - if !host.contains(':') { - return Ok((host.to_owned(), parse_port(port)?)); - } - } - - Ok((host_port.to_owned(), default_port)) -} - -fn parse_port(port: &str) -> Result { - port.parse().map_err(|_| RtspVideoSourceError::InvalidUrl("invalid port")) -} - /// A parsed RTSP response. #[derive(Debug, Clone)] pub(super) struct RtspResponse { From 4d9ee79d85ff6dca667f9e3abeddde56e3a3b93f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:44:21 -0700 Subject: [PATCH 24/38] Reuse codec helpers --- livekit-capture/src/encoded/h26x.rs | 6 +++-- .../src/sources/rtsp/dimensions.rs | 9 ++++--- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 26 +++++++++---------- 3 files changed, 23 insertions(+), 18 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 2790a0db4..9402966ff 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -481,12 +481,14 @@ fn is_keyframe_nalus( } } -fn h264_nal_type(nal: &[u8]) -> Result { +/// Extracts the type of an H.264 NAL unit from its header. +pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { let header = nal.first().ok_or(H26xParseError::EmptyPayload)?; Ok(header & 0x1f) } -fn h265_nal_type(nal: &[u8]) -> Result { +/// Extracts the type of an H.265 NAL unit from its header. +pub(crate) fn h265_nal_type(nal: &[u8]) -> Result { if nal.is_empty() { return Err(H26xParseError::EmptyPayload); } diff --git a/livekit-capture/src/sources/rtsp/dimensions.rs b/livekit-capture/src/sources/rtsp/dimensions.rs index 08e044fa6..6ccf712c4 100644 --- a/livekit-capture/src/sources/rtsp/dimensions.rs +++ b/livekit-capture/src/sources/rtsp/dimensions.rs @@ -19,7 +19,10 @@ use super::bits::{read_leb128, BitReader}; use crate::{ - encoded::{h26x::annex_b_nalus, EncodedVideoCodec}, + encoded::{ + h26x::{annex_b_nalus, h264_nal_type, h265_nal_type}, + EncodedVideoCodec, + }, primitive::VideoResolution, }; @@ -31,11 +34,11 @@ pub(super) fn access_unit_resolution( match codec { EncodedVideoCodec::H264 => annex_b_nalus(payload) .into_iter() - .find(|nal| nal.first().is_some_and(|header| header & 0x1f == 7)) + .find(|nal| matches!(h264_nal_type(nal), Ok(7))) .and_then(|nal| sps_resolution(codec, nal)), EncodedVideoCodec::H265 => annex_b_nalus(payload) .into_iter() - .find(|nal| nal.first().is_some_and(|header| (header >> 1) & 0x3f == 33)) + .find(|nal| matches!(h265_nal_type(nal), Ok(33))) .and_then(|nal| sps_resolution(codec, nal)), EncodedVideoCodec::VP8 => vp8_keyframe_resolution(payload), EncodedVideoCodec::VP9 => vp9_keyframe_resolution(payload), diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs index f9e668810..144ca75d1 100644 --- a/livekit-capture/src/sources/rtsp/rtp/h26x.rs +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -15,7 +15,10 @@ //! H.264 (RFC 6184) and H.265 (RFC 7798) RTP payload handling. use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; -use crate::encoded::{h26x::access_unit_from_nalus, EncodedFrameType, EncodedVideoCodec}; +use crate::encoded::{ + h26x::{access_unit_from_nalus, h264_nal_type, h265_nal_type}, + EncodedFrameType, EncodedVideoCodec, +}; impl RtpAccessUnitAssembler { pub(super) fn push_h264_payload( @@ -232,21 +235,18 @@ struct NalPresence { impl NalPresence { fn record(&mut self, codec: EncodedVideoCodec, nal: &[u8]) { - let Some(&header) = nal.first() else { - return; - }; match codec { - EncodedVideoCodec::H264 => match header & 0x1f { - 5 => self.idr = true, - 7 => self.sps = true, - 8 => self.pps = true, + EncodedVideoCodec::H264 => match h264_nal_type(nal) { + Ok(5) => self.idr = true, + Ok(7) => self.sps = true, + Ok(8) => self.pps = true, _ => {} }, - EncodedVideoCodec::H265 => match (header >> 1) & 0x3f { - 19 | 20 => self.idr = true, - 32 => self.vps = true, - 33 => self.sps = true, - 34 => self.pps = true, + EncodedVideoCodec::H265 => match h265_nal_type(nal) { + Ok(19 | 20) => self.idr = true, + Ok(32) => self.vps = true, + Ok(33) => self.sps = true, + Ok(34) => self.pps = true, _ => {} }, _ => {} From ed12bf95ffbb79f290654bd134efec4133381816 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:48:07 -0700 Subject: [PATCH 25/38] Reuse AVC NAL splitter --- livekit-capture/src/encoded/h26x.rs | 4 ++- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 29 ++++++++++---------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 9402966ff..ee282fa9d 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -637,7 +637,9 @@ fn find_start_code(bytes: &[u8]) -> Option<(usize, usize)> { None } -fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { +/// Splits a sequence of length-prefixed NAL units, as used by AVC-format +/// access units and, with a 2-byte length, RTP aggregation packets. +pub(crate) fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { let ranges = avc_nal_ranges(payload, nal_length_size, true)?; if ranges.is_empty() { return Err(H26xParseError::EmptyPayload); diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs index 144ca75d1..496e3fbbc 100644 --- a/livekit-capture/src/sources/rtsp/rtp/h26x.rs +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -16,7 +16,7 @@ use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; use crate::encoded::{ - h26x::{access_unit_from_nalus, h264_nal_type, h265_nal_type}, + h26x::{access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type}, EncodedFrameType, EncodedVideoCodec, }; @@ -63,24 +63,17 @@ impl RtpAccessUnitAssembler { /// Unpacks the length-prefixed NAL units of an H.264 STAP-A or H.265 AP /// payload, whose aggregation headers the caller has already stripped. + /// + /// The layout is the same 2-byte-length-prefixed sequence as an + /// AVC-format access unit, so the AVC splitter does the walking. fn push_h26x_aggregation( &mut self, rtp_timestamp: u32, payload: &[u8], ) -> Result<(), RtpDepacketizerError> { - let mut cursor = 0; - while cursor < payload.len() { - if payload.len() < cursor + 2 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - let len = u16::from_be_bytes([payload[cursor], payload[cursor + 1]]) as usize; - cursor += 2; - if len == 0 || payload.len() < cursor + len { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - self.current_mut(rtp_timestamp)?.nal_units.push(payload[cursor..cursor + len].to_vec()); - cursor += len; - } + let nal_units: Vec> = + avc_nalus(payload, 2)?.into_iter().map(<[u8]>::to_vec).collect(); + self.current_mut(rtp_timestamp)?.nal_units.extend(nal_units); Ok(()) } @@ -307,6 +300,14 @@ mod tests { ); } + #[test] + fn rejects_empty_h264_aggregation() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // A STAP-A with no NAL entries is malformed. + let empty_stap = rtp_packet(10, 12_000, true, &[0x18]); + assert!(assembler.push(&empty_stap).is_err()); + } + #[test] fn assembles_h264_fu_a() { let mut assembler = assembler(EncodedVideoCodec::H264); From 2576c707cef2074f7f5ec8176aa266730dbbb57f Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:01:07 -0700 Subject: [PATCH 26/38] Bound access unit assembly --- livekit-capture/src/encoded/h26x.rs | 2 +- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 3 + livekit-capture/src/sources/rtsp/rtp/mod.rs | 84 +++++++++++++++++++- 3 files changed, 87 insertions(+), 2 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index ee282fa9d..982eee423 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -44,7 +44,7 @@ pub enum H26xParseError { const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; /// Upper bound on bytes buffered while waiting for an access-unit boundary. -const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; +pub(crate) const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; /// Byte-stream access-unit parser shared by the encoded ingest sources. /// diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs index 496e3fbbc..34415cee1 100644 --- a/livekit-capture/src/sources/rtsp/rtp/h26x.rs +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -170,6 +170,9 @@ impl RtpAccessUnitAssembler { /// passthrough subscribers may join mid-stream and can only initialize /// their decoder from parameter sets inside the keyframe itself. pub(super) fn finish_current(&mut self) -> Result<(), RtpDepacketizerError> { + // An open fragment carrying into the next unit undercounts by at + // most one NAL's fragment, which the generous cap absorbs. + self.pending_bytes = 0; let Some(current) = self.current.take() else { return Ok(()); }; diff --git a/livekit-capture/src/sources/rtsp/rtp/mod.rs b/livekit-capture/src/sources/rtsp/rtp/mod.rs index b72c7228a..c83b7aa8a 100644 --- a/livekit-capture/src/sources/rtsp/rtp/mod.rs +++ b/livekit-capture/src/sources/rtsp/rtp/mod.rs @@ -28,7 +28,8 @@ use thiserror::Error; use crate::{ encoded::{ - h26x::H26xParseError, EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + h26x::{H26xParseError, MAX_PENDING_ACCESS_UNIT_BYTES}, + EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, }, primitive::VideoResolution, }; @@ -211,8 +212,12 @@ pub(super) struct RtpAccessUnitAssembler { av1_fragment: Option, ready: VecDeque, awaiting_keyframe: bool, + /// Payload bytes accumulated since the last completed or discarded + /// access unit, bounding what an endless unit can buffer. + pending_bytes: usize, logged_ssrc_mismatch: bool, warned_missing_parameter_sets: bool, + warned_oversized_pending: bool, sequence_gaps: u64, dropped_access_units: u64, } @@ -277,8 +282,10 @@ impl RtpAccessUnitAssembler { av1_fragment: None, ready: VecDeque::new(), awaiting_keyframe: false, + pending_bytes: 0, logged_ssrc_mismatch: false, warned_missing_parameter_sets: false, + warned_oversized_pending: false, sequence_gaps: 0, dropped_access_units: 0, }) @@ -338,6 +345,25 @@ impl RtpAccessUnitAssembler { EncodedVideoCodec::AV1 => self.push_av1_payload(&packet)?, } + // Bound what an unfinished access unit can buffer: a stream that + // never completes one must not grow memory without limit. Counting + // raw payload bytes slightly overestimates, which only trips the + // generous cap earlier. + self.pending_bytes = self.pending_bytes.saturating_add(packet.payload.len()); + if self.pending_bytes > MAX_PENDING_ACCESS_UNIT_BYTES { + if !self.warned_oversized_pending { + self.warned_oversized_pending = true; + log::warn!( + "discarding an access unit still incomplete after \ + {MAX_PENDING_ACCESS_UNIT_BYTES} buffered bytes; \ + waiting for the next keyframe" + ); + } + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + if packet.marker { if self.fragment.is_some() || self.av1_fragment.is_some() { // The marker closed the access unit before the open fragment's @@ -411,6 +437,7 @@ impl RtpAccessUnitAssembler { self.fragment = None; self.current_frame = None; self.av1_fragment = None; + self.pending_bytes = 0; self.awaiting_keyframe = true; } @@ -475,6 +502,9 @@ impl RtpAccessUnitAssembler { /// Completes the pending VP8/VP9/AV1 frame and queues it. fn finish_current_frame(&mut self) -> Result<(), RtpDepacketizerError> { + // An open fragment carrying into the next unit undercounts by at + // most one frame's fragment, which the generous cap absorbs. + self.pending_bytes = 0; let Some(current) = self.current_frame.take() else { return Ok(()); }; @@ -624,6 +654,58 @@ mod tests { assert_eq!(assembler.stats().sequence_gaps, 0); } + #[test] + fn caps_h264_pending_access_unit_bytes() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let chunk = [0xaa; 60_000]; + + // An FU-A start followed by endless continuations at one timestamp: + // the fragment never completes, so buffering must stop at the cap. + let mut payload = vec![0x7c, 0x85]; + payload.extend_from_slice(&chunk); + assert!(push_one(&mut assembler, &rtp_packet(0, 12_000, false, &payload)).is_none()); + + let mut payload = vec![0x7c, 0x05]; + payload.extend_from_slice(&chunk); + let mut sequence_number = 1u16; + while !assembler.stats().awaiting_keyframe { + assert!(sequence_number < 1_000, "the pending byte cap never triggered"); + let packet = rtp_packet(sequence_number, 12_000, false, &payload); + assert!(push_one(&mut assembler, &packet).is_none()); + sequence_number += 1; + } + assert!(assembler.stats().dropped_access_units >= 1); + + // The stream recovers at the next keyframe. + let key = rtp_packet(sequence_number, 15_000, true, &[0x65, 1, 2]); + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + } + + #[test] + fn caps_vp8_pending_frame_bytes() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let chunk = [0xaa; 60_000]; + + let mut payload = vec![0x10, 0x00]; + payload.extend_from_slice(&chunk); + assert!(push_one(&mut assembler, &rtp_packet(0, 12_000, false, &payload)).is_none()); + + let mut payload = vec![0x00]; + payload.extend_from_slice(&chunk); + let mut sequence_number = 1u16; + while !assembler.stats().awaiting_keyframe { + assert!(sequence_number < 1_000, "the pending byte cap never triggered"); + let packet = rtp_packet(sequence_number, 12_000, false, &payload); + assert!(push_one(&mut assembler, &packet).is_none()); + sequence_number += 1; + } + + let key = rtp_packet(sequence_number, 15_000, true, &[0x10, 0x00, 1, 2]); + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + } + #[test] fn timestamp_change_completes_marker_less_access_unit() { let mut assembler = assembler(EncodedVideoCodec::H264); From c667b5086ed8eaa860ba3f7a511a875f5238f45e Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:05:02 -0700 Subject: [PATCH 27/38] Add byte reader --- livekit-capture/src/sources/rtsp/bits.rs | 97 +++++++++++++++++++ livekit-capture/src/sources/rtsp/rtp/av1.rs | 99 +++++++------------ livekit-capture/src/sources/rtsp/rtp/mod.rs | 66 ++++++------- livekit-capture/src/sources/rtsp/rtp/vpx.rs | 101 +++++++------------- 4 files changed, 196 insertions(+), 167 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/bits.rs b/livekit-capture/src/sources/rtsp/bits.rs index 2cfdd7b2f..c70182b4e 100644 --- a/livekit-capture/src/sources/rtsp/bits.rs +++ b/livekit-capture/src/sources/rtsp/bits.rs @@ -82,6 +82,81 @@ impl<'a> BitReader<'a> { } } +/// Byte-level reader over untrusted input. Every read returns `None` past +/// the end of the input, so callers cannot index out of bounds. +#[derive(Debug, Clone)] +pub(super) struct ByteReader<'a> { + bytes: &'a [u8], +} + +impl<'a> ByteReader<'a> { + /// Creates a reader positioned at the first byte. + pub(super) fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } + + /// Returns `true` when all input was consumed. + pub(super) fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Reads one byte. + pub(super) fn get_u8(&mut self) -> Option { + let (&byte, rest) = self.bytes.split_first()?; + self.bytes = rest; + Some(byte) + } + + /// Reads a big-endian `u16`. + pub(super) fn get_u16_be(&mut self) -> Option { + let bytes = self.take(2)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) + } + + /// Reads a big-endian `u32`. + pub(super) fn get_u32_be(&mut self) -> Option { + let bytes = self.take(4)?; + Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + /// Reads an AV1/LEB128-encoded length. + pub(super) fn get_leb128(&mut self) -> Option { + let mut value = 0usize; + let mut shift = 0usize; + loop { + let byte = self.get_u8()?; + value |= usize::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Some(value); + } + shift += 7; + if shift >= usize::BITS as usize { + return None; + } + } + } + + /// Reads the next `len` bytes. + pub(super) fn take(&mut self, len: usize) -> Option<&'a [u8]> { + if len > self.bytes.len() { + return None; + } + let (taken, rest) = self.bytes.split_at(len); + self.bytes = rest; + Some(taken) + } + + /// Skips over the next `len` bytes. + pub(super) fn skip(&mut self, len: usize) -> Option<()> { + self.take(len).map(|_| ()) + } + + /// Consumes and returns the unread remainder. + pub(super) fn take_rest(&mut self) -> &'a [u8] { + std::mem::take(&mut self.bytes) + } +} + /// Reads an AV1/LEB128 length from `bytes` at `cursor`, advancing it. pub(super) fn read_leb128(bytes: &[u8], cursor: &mut usize) -> Option { let mut value = 0usize; @@ -145,6 +220,28 @@ mod tests { assert_eq!(reader.read_ue(), None); } + #[test] + fn byte_reader_reads_and_bounds() { + let mut reader = ByteReader::new(&[1, 0, 2, 0, 0, 0, 3, 4, 5, 6]); + assert_eq!(reader.get_u8(), Some(1)); + assert_eq!(reader.get_u16_be(), Some(2)); + assert_eq!(reader.get_u32_be(), Some(3)); + assert_eq!(reader.take(2), Some(&[4, 5][..])); + assert!(reader.skip(2).is_none()); + assert_eq!(reader.take_rest(), &[6]); + assert!(reader.is_empty()); + assert_eq!(reader.get_u8(), None); + } + + #[test] + fn byte_reader_reads_leb128() { + let mut reader = ByteReader::new(&[0xac, 0x02, 0x00]); + assert_eq!(reader.get_leb128(), Some(300)); + assert_eq!(reader.get_leb128(), Some(0)); + assert_eq!(reader.get_leb128(), None); + assert_eq!(ByteReader::new(&[0x80]).get_leb128(), None); + } + #[test] fn leb128_round_trips() { for value in [0usize, 1, 127, 128, 300, 16_383, 16_384, usize::from(u16::MAX)] { diff --git a/livekit-capture/src/sources/rtsp/rtp/av1.rs b/livekit-capture/src/sources/rtsp/rtp/av1.rs index 54b2e4541..3f53ae4a9 100644 --- a/livekit-capture/src/sources/rtsp/rtp/av1.rs +++ b/livekit-capture/src/sources/rtsp/rtp/av1.rs @@ -20,7 +20,7 @@ use super::{Av1FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; use crate::{ encoded::EncodedFrameType, - sources::rtsp::bits::{read_leb128, write_leb128, BitReader}, + sources::rtsp::bits::{write_leb128, BitReader, ByteReader}, }; impl RtpAccessUnitAssembler { @@ -89,44 +89,29 @@ struct Av1PayloadDescriptor<'a> { fn parse_av1_payload_descriptor( payload: &[u8], ) -> Result, RtpDepacketizerError> { - let Some(&header) = payload.first() else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let header = reader.get_u8().ok_or_else(malformed)?; let starts_fragment = header & 0x80 != 0; let ends_fragment = header & 0x40 != 0; - let element_count = (header >> 4) & 0x03; + let element_count = usize::from((header >> 4) & 0x03); - let mut cursor = 1; let mut elements = Vec::new(); if element_count == 0 { - while cursor < payload.len() { - let len = read_leb128(payload, &mut cursor) - .ok_or(RtpDepacketizerError::UnsupportedPayload)?; - let Some(end) = cursor.checked_add(len) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - if end > payload.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - elements.push(&payload[cursor..end]); - cursor = end; + while !reader.is_empty() { + let len = reader.get_leb128().ok_or_else(malformed)?; + elements.push(reader.take(len).ok_or_else(malformed)?); } } else { - for index in 0..usize::from(element_count) { - let len = if index + 1 == usize::from(element_count) { - payload.len().saturating_sub(cursor) + for index in 0..element_count { + let element = if index + 1 == element_count { + reader.take_rest() } else { - read_leb128(payload, &mut cursor) - .ok_or(RtpDepacketizerError::UnsupportedPayload)? - }; - let Some(end) = cursor.checked_add(len) else { - return Err(RtpDepacketizerError::UnsupportedPayload); + let len = reader.get_leb128().ok_or_else(malformed)?; + reader.take(len).ok_or_else(malformed)? }; - if end > payload.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - elements.push(&payload[cursor..end]); - cursor = end; + elements.push(element); } } @@ -135,39 +120,34 @@ fn parse_av1_payload_descriptor( /// Converts an RTP OBU element into a size-prefixed OBU. fn av1_obu_from_rtp_element(element: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; let Some(&header) = element.first() else { - return Err(RtpDepacketizerError::UnsupportedPayload); + return Err(malformed()); }; if header & 0x80 != 0 { - return Err(RtpDepacketizerError::UnsupportedPayload); + return Err(malformed()); } + let header_len = if header & 0x04 != 0 { 2 } else { 1 }; if header & 0x02 != 0 { - let mut cursor = if header & 0x04 != 0 { 2 } else { 1 }; - if cursor > element.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - let payload_size = - read_leb128(element, &mut cursor).ok_or(RtpDepacketizerError::UnsupportedPayload)?; - if payload_size != element.len().saturating_sub(cursor) { - return Err(RtpDepacketizerError::UnsupportedPayload); + // Already size-prefixed; validate the size against the element. + let mut reader = ByteReader::new(element); + reader.skip(header_len).ok_or_else(malformed)?; + let payload_size = reader.get_leb128().ok_or_else(malformed)?; + if payload_size != reader.take_rest().len() { + return Err(malformed()); } return Ok(element.to_vec()); } - let payload_offset = if header & 0x04 != 0 { 2 } else { 1 }; - if payload_offset > element.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - - let payload_size = element.len() - payload_offset; + let payload = element.get(header_len..).ok_or_else(malformed)?; let mut obu = Vec::with_capacity(element.len() + 8); obu.push(header | 0x02); if header & 0x04 != 0 { obu.push(element[1]); } - write_leb128(payload_size, &mut obu); - obu.extend_from_slice(&element[payload_offset..]); + write_leb128(payload.len(), &mut obu); + obu.extend_from_slice(payload); Ok(obu) } @@ -214,34 +194,27 @@ fn av1_frame_type( /// Splits an OBU into its type and payload bytes. fn av1_obu_parts(obu: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; let Some(&header) = obu.first() else { return Ok(None); }; if header & 0x80 != 0 { - return Err(RtpDepacketizerError::UnsupportedPayload); + return Err(malformed()); } let obu_type = (header & 0x78) >> 3; let has_extension = header & 0x04 != 0; let has_size = header & 0x02 != 0; - let mut cursor = if has_extension { 2 } else { 1 }; - if cursor > obu.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } + let mut reader = ByteReader::new(obu); + reader.skip(if has_extension { 2 } else { 1 }).ok_or_else(malformed)?; if !has_size { - return Ok(Some((obu_type, &obu[cursor..]))); + return Ok(Some((obu_type, reader.take_rest()))); } - let payload_size = - read_leb128(obu, &mut cursor).ok_or(RtpDepacketizerError::UnsupportedPayload)?; - let Some(end) = cursor.checked_add(payload_size) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - if end > obu.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - Ok(Some((obu_type, &obu[cursor..end]))) + let payload_size = reader.get_leb128().ok_or_else(malformed)?; + let payload = reader.take(payload_size).ok_or_else(malformed)?; + Ok(Some((obu_type, payload))) } #[cfg(test)] diff --git a/livekit-capture/src/sources/rtsp/rtp/mod.rs b/livekit-capture/src/sources/rtsp/rtp/mod.rs index c83b7aa8a..a81cd8071 100644 --- a/livekit-capture/src/sources/rtsp/rtp/mod.rs +++ b/livekit-capture/src/sources/rtsp/rtp/mod.rs @@ -32,6 +32,7 @@ use crate::{ EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, }, primitive::VideoResolution, + sources::rtsp::bits::ByteReader, }; /// Out-of-band H.26x parameter sets, decoded from SDP `fmtp` attributes. @@ -77,53 +78,48 @@ pub(super) struct RtpPacket<'a> { impl<'a> RtpPacket<'a> { /// Parses a single RTP packet. pub(super) fn parse(bytes: &'a [u8]) -> Result { + let too_short = || RtpDepacketizerError::PacketTooShort; if bytes.len() < 12 { - return Err(RtpDepacketizerError::PacketTooShort); - } - if bytes[0] >> 6 != 2 { - return Err(RtpDepacketizerError::UnsupportedVersion(bytes[0] >> 6)); + return Err(too_short()); } + let mut reader = ByteReader::new(bytes); - let has_padding = (bytes[0] & 0x20) != 0; - let has_extension = (bytes[0] & 0x10) != 0; - let csrc_count = (bytes[0] & 0x0f) as usize; - let mut payload_start = 12 + csrc_count * 4; - if bytes.len() < payload_start { - return Err(RtpDepacketizerError::PacketTooShort); + let flags = reader.get_u8().ok_or_else(too_short)?; + if flags >> 6 != 2 { + return Err(RtpDepacketizerError::UnsupportedVersion(flags >> 6)); } + let has_padding = (flags & 0x20) != 0; + let has_extension = (flags & 0x10) != 0; + let csrc_count = (flags & 0x0f) as usize; + + let marker_and_type = reader.get_u8().ok_or_else(too_short)?; + let sequence_number = reader.get_u16_be().ok_or_else(too_short)?; + let timestamp = reader.get_u32_be().ok_or_else(too_short)?; + let ssrc = reader.get_u32_be().ok_or_else(too_short)?; + reader.skip(csrc_count * 4).ok_or_else(too_short)?; if has_extension { - if bytes.len() < payload_start + 4 { - return Err(RtpDepacketizerError::PacketTooShort); - } - let extension_words = - u16::from_be_bytes([bytes[payload_start + 2], bytes[payload_start + 3]]) as usize; - payload_start += 4 + extension_words * 4; - if bytes.len() < payload_start { - return Err(RtpDepacketizerError::PacketTooShort); - } + reader.skip(2).ok_or_else(too_short)?; // profile-specific identifier + let extension_words = reader.get_u16_be().ok_or_else(too_short)? as usize; + reader.skip(extension_words * 4).ok_or_else(too_short)?; } - let payload_end = if has_padding { - let Some(padding) = bytes.last().copied() else { - return Err(RtpDepacketizerError::PacketTooShort); - }; - let padding = padding as usize; - if padding == 0 || bytes.len() < payload_start + padding { + let mut payload = reader.take_rest(); + if has_padding { + let padding = *payload.last().ok_or_else(too_short)? as usize; + if padding == 0 || padding > payload.len() { return Err(RtpDepacketizerError::PacketTooShort); } - bytes.len() - padding - } else { - bytes.len() - }; + payload = &payload[..payload.len() - padding]; + } Ok(Self { - marker: (bytes[1] & 0x80) != 0, - payload_type: bytes[1] & 0x7f, - sequence_number: u16::from_be_bytes([bytes[2], bytes[3]]), - timestamp: u32::from_be_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]), - ssrc: u32::from_be_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]), - payload: &bytes[payload_start..payload_end], + marker: (marker_and_type & 0x80) != 0, + payload_type: marker_and_type & 0x7f, + sequence_number, + timestamp, + ssrc, + payload, }) } } diff --git a/livekit-capture/src/sources/rtsp/rtp/vpx.rs b/livekit-capture/src/sources/rtsp/rtp/vpx.rs index 28407defb..d96511d3e 100644 --- a/livekit-capture/src/sources/rtsp/rtp/vpx.rs +++ b/livekit-capture/src/sources/rtsp/rtp/vpx.rs @@ -18,7 +18,7 @@ //! with [`RtpDepacketizerError::UnsupportedPayloadDescriptor`]. use super::{RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; -use crate::encoded::EncodedFrameType; +use crate::{encoded::EncodedFrameType, sources::rtsp::bits::ByteReader}; impl RtpAccessUnitAssembler { pub(super) fn push_vp8_payload( @@ -100,92 +100,77 @@ struct Vp9PayloadDescriptor<'a> { fn parse_vp8_payload_descriptor( payload: &[u8], ) -> Result, RtpDepacketizerError> { - let Some(&descriptor) = payload.first() else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let descriptor = reader.get_u8().ok_or_else(malformed)?; let start_of_partition = descriptor & 0x10 != 0; let partition_id = descriptor & 0x0f; - let mut cursor = 1; if descriptor & 0x80 != 0 { - let Some(&extension) = payload.get(cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - cursor += 1; + let extension = reader.get_u8().ok_or_else(malformed)?; if extension & 0x80 != 0 { - let Some(&picture_id) = payload.get(cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - cursor += if picture_id & 0x80 != 0 { 2 } else { 1 }; + let picture_id = reader.get_u8().ok_or_else(malformed)?; + if picture_id & 0x80 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } } if extension & 0x40 != 0 { - cursor += 1; + reader.skip(1).ok_or_else(malformed)?; } if extension & 0x20 != 0 || extension & 0x10 != 0 { - cursor += 1; + reader.skip(1).ok_or_else(malformed)?; } } - if cursor > payload.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - Ok(Vp8PayloadDescriptor { start_of_partition, partition_id, payload: &payload[cursor..] }) + Ok(Vp8PayloadDescriptor { start_of_partition, partition_id, payload: reader.take_rest() }) } fn parse_vp9_payload_descriptor( payload: &[u8], ) -> Result, RtpDepacketizerError> { - let Some(&descriptor) = payload.first() else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let descriptor = reader.get_u8().ok_or_else(malformed)?; if descriptor & 0x10 != 0 { return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); } let beginning_of_frame = descriptor & 0x08 != 0; let inter_picture_predicted = descriptor & 0x40 != 0; - let mut cursor = 1; if descriptor & 0x80 != 0 { - let Some(&picture_id) = payload.get(cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - cursor += if picture_id & 0x80 != 0 { 2 } else { 1 }; + let picture_id = reader.get_u8().ok_or_else(malformed)?; + if picture_id & 0x80 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } } let mut spatial_id = None; let mut inter_layer_predicted = None; if descriptor & 0x20 != 0 { - let Some(&layer_info) = payload.get(cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - cursor += 1; + let layer_info = reader.get_u8().ok_or_else(malformed)?; spatial_id = Some((layer_info >> 1) & 0x07); inter_layer_predicted = Some(layer_info & 0x01 != 0); - cursor += 1; // TL0PICIDX is present in non-flexible mode. + reader.skip(1).ok_or_else(malformed)?; // TL0PICIDX in non-flexible mode } if descriptor & 0x02 != 0 { - skip_vp9_scalability_structure(payload, &mut cursor)?; + skip_vp9_scalability_structure(&mut reader)?; } - if cursor > payload.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } Ok(Vp9PayloadDescriptor { beginning_of_frame, inter_picture_predicted, spatial_id, inter_layer_predicted, - payload: &payload[cursor..], + payload: reader.take_rest(), }) } fn skip_vp9_scalability_structure( - payload: &[u8], - cursor: &mut usize, + reader: &mut ByteReader<'_>, ) -> Result<(), RtpDepacketizerError> { - let Some(&structure) = payload.get(*cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - *cursor += 1; + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let structure = reader.get_u8().ok_or_else(malformed)?; let spatial_layers = ((structure >> 5) & 0x07) + 1; if spatial_layers != 1 { @@ -193,42 +178,20 @@ fn skip_vp9_scalability_structure( } if structure & 0x10 != 0 { - let bytes = usize::from(spatial_layers) * 4; - skip_bytes(payload, cursor, bytes)?; + reader.skip(usize::from(spatial_layers) * 4).ok_or_else(malformed)?; } if structure & 0x08 != 0 { - let Some(&group_count) = payload.get(*cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - *cursor += 1; + let group_count = reader.get_u8().ok_or_else(malformed)?; for _ in 0..group_count { - let Some(&group) = payload.get(*cursor) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - *cursor += 1; - skip_bytes(payload, cursor, usize::from((group >> 2) & 0x03))?; + let group = reader.get_u8().ok_or_else(malformed)?; + reader.skip(usize::from((group >> 2) & 0x03)).ok_or_else(malformed)?; } } Ok(()) } -fn skip_bytes( - payload: &[u8], - cursor: &mut usize, - bytes: usize, -) -> Result<(), RtpDepacketizerError> { - let Some(next) = cursor.checked_add(bytes) else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - if next > payload.len() { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - *cursor = next; - Ok(()) -} - fn is_vp8_keyframe(payload: &[u8]) -> bool { payload.first().is_some_and(|header| header & 0x01 == 0) } From ec086f6bce6f2d86c0d0e2cfdc89bcf9c4bf6add Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:05:47 -0700 Subject: [PATCH 28/38] Strip control characters from server strings --- livekit-capture/src/sources/rtsp/client.rs | 15 ++++++++++++++- livekit-capture/src/sources/rtsp/sdp.rs | 3 ++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index b77949ffa..71022febb 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -139,7 +139,10 @@ impl RtspResponse { fn from_message(response: rtsp_types::Response>) -> Self { Self { status_code: response.status().into(), - reason: response.reason_phrase().to_owned(), + // The reason phrase flows into error strings and logs; strip + // control characters so a server cannot forge log lines or + // emit terminal escapes. + reason: response.reason_phrase().chars().filter(|c| !c.is_control()).collect(), headers: response .headers() .map(|(name, value)| (name.as_str().to_owned(), value.as_str().to_owned())) @@ -857,6 +860,16 @@ mod tests { assert!(response.is_success()); } + #[test] + fn strips_control_characters_from_reason_phrases() { + let message = rtsp_types::Response::builder(Version::V1_0, rtsp_types::StatusCode::Ok) + .reason_phrase("OK\u{1b}[31m\nfake log line") + .build(Vec::new()); + + let response = RtspResponse::from_message(message); + assert_eq!(response.reason, "OK[31mfake log line"); + } + #[test] fn parses_session_header() { assert_eq!(parse_session_id("abc123;timeout=60").unwrap(), "abc123"); diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs index 7b43a5ad9..6742d199a 100644 --- a/livekit-capture/src/sources/rtsp/sdp.rs +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -68,7 +68,8 @@ pub(super) fn parse_sdp_session( expected_codec: Option, ) -> Result { let session = sdp_types::Session::parse(sdp).map_err(|err| { - log::debug!("failed to parse SDP: {err}"); + // The parser error can quote server-provided bytes; escape them. + log::debug!("failed to parse SDP: {}", err.to_string().escape_debug()); RtspVideoSourceError::InvalidSdp })?; let session_control = attribute_value(&session.attributes, "control"); From 005eecce11bc445ec57bfdd398a860a6b5345c4a Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:06:37 -0700 Subject: [PATCH 29/38] Warn when TLS verification is disabled --- livekit-capture/src/sources/rtsp/mod.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index 6f1cd20f9..65c06edd4 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -317,6 +317,13 @@ impl RtspVideoSource { if url.tls { return Err(RtspVideoSourceError::TlsNotSupported); } + if url.tls && config.accept_invalid_tls_certs { + log::warn!( + "TLS certificate verification is disabled for {}; \ + the connection is encrypted but the server is not authenticated", + url.request_uri, + ); + } let credentials = merge_credentials(&config, &url); let connect_timeout = duration_ms(config.connect_timeout_ms, DEFAULT_CONNECT_TIMEOUT); let idle_timeout = duration_ms(config.idle_timeout_ms, DEFAULT_IDLE_TIMEOUT); From 01790c9d83c7a1a7fa37a576d7a6e803f1e4d3af Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:32:29 -0700 Subject: [PATCH 30/38] Split payload parsing from assembly --- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 312 +++++++++++++------ 1 file changed, 210 insertions(+), 102 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs index 34415cee1..4a85c5c38 100644 --- a/livekit-capture/src/sources/rtsp/rtp/h26x.rs +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -15,29 +15,137 @@ //! H.264 (RFC 6184) and H.265 (RFC 7798) RTP payload handling. use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; -use crate::encoded::{ - h26x::{access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type}, - EncodedFrameType, EncodedVideoCodec, +use crate::{ + encoded::{ + h26x::{access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type}, + EncodedFrameType, EncodedVideoCodec, + }, + sources::rtsp::bits::ByteReader, }; +/// One parsed H.264 RTP payload (RFC 6184). +#[derive(Debug, PartialEq, Eq)] +enum H264Payload<'a> { + /// A complete NAL unit (types 1-23), including its header. + Nal(&'a [u8]), + /// STAP-A: aggregated complete NAL units. + Aggregation(Vec<&'a [u8]>), + /// FU-A: a fragment of one NAL unit. + Fragment(Fragment<'a>), +} + +/// One parsed H.265 RTP payload (RFC 7798). +#[derive(Debug, PartialEq, Eq)] +enum H265Payload<'a> { + /// A complete NAL unit (types 0-47), including its header. + Nal(&'a [u8]), + /// AP: aggregated complete NAL units. + Aggregation(Vec<&'a [u8]>), + /// FU: a fragment of one NAL unit. + Fragment(Fragment<'a>), +} + +/// A fragment of one NAL unit. +#[derive(Debug, PartialEq, Eq)] +struct Fragment<'a> { + /// Whether this is the first fragment of the NAL unit. + start: bool, + /// Whether this is the last fragment of the NAL unit. + end: bool, + /// The fragmented NAL unit's header, reconstructed from the + /// fragmentation headers. + nal_header: NalHeader, + /// Fragment payload bytes. + payload: &'a [u8], +} + +/// A reconstructed NAL unit header. +#[derive(Debug, PartialEq, Eq)] +enum NalHeader { + H264(u8), + H265([u8; 2]), +} + +impl NalHeader { + fn as_slice(&self) -> &[u8] { + match self { + Self::H264(header) => std::slice::from_ref(header), + Self::H265(header) => header, + } + } +} + +/// Parses one H.264 RTP payload without touching assembly state. +fn parse_h264_payload(payload: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + let indicator = reader.get_u8().ok_or_else(malformed)?; + + match indicator & 0x1f { + 1..=23 => Ok(H264Payload::Nal(payload)), + // STAP-A: 2-byte-length-prefixed NAL units, like an AVC access unit. + 24 => Ok(H264Payload::Aggregation(avc_nalus(reader.take_rest(), 2)?)), + 28 => { + let fu_header = reader.get_u8().ok_or_else(malformed)?; + let nal_type = fu_header & 0x1f; + if nal_type == 0 || nal_type > 23 { + return Err(malformed()); + } + Ok(H264Payload::Fragment(Fragment { + start: fu_header & 0x80 != 0, + end: fu_header & 0x40 != 0, + nal_header: NalHeader::H264((indicator & 0xe0) | nal_type), + payload: reader.take_rest(), + })) + } + _ => Err(malformed()), + } +} + +/// Parses one H.265 RTP payload without touching assembly state. +fn parse_h265_payload(payload: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + let header = [ + reader.get_u8().ok_or_else(malformed)?, + reader.get_u8().ok_or_else(malformed)?, + ]; + + match (header[0] >> 1) & 0x3f { + 0..=47 => Ok(H265Payload::Nal(payload)), + // AP: 2-byte-length-prefixed NAL units, like an AVC access unit. + 48 => Ok(H265Payload::Aggregation(avc_nalus(reader.take_rest(), 2)?)), + 49 => { + let fu_header = reader.get_u8().ok_or_else(malformed)?; + let nal_type = fu_header & 0x3f; + if nal_type > 47 { + return Err(malformed()); + } + Ok(H265Payload::Fragment(Fragment { + start: fu_header & 0x80 != 0, + end: fu_header & 0x40 != 0, + nal_header: NalHeader::H265([(header[0] & 0x81) | (nal_type << 1), header[1]]), + payload: reader.take_rest(), + })) + } + _ => Err(malformed()), + } +} + impl RtpAccessUnitAssembler { pub(super) fn push_h264_payload( &mut self, packet: &RtpPacket<'_>, ) -> Result<(), RtpDepacketizerError> { - let payload = packet.payload; - let Some(&header) = payload.first() else { - return Err(RtpDepacketizerError::UnsupportedPayload); - }; - let nal_type = header & 0x1f; - - match nal_type { - 1..=23 => self.current_mut(packet.timestamp)?.nal_units.push(payload.to_vec()), - 24 => self.push_h26x_aggregation(packet.timestamp, &payload[1..])?, - 28 => self.push_h264_fu_a(packet.timestamp, payload)?, - _ => return Err(RtpDepacketizerError::UnsupportedPayload), + match parse_h264_payload(packet.payload)? { + H264Payload::Nal(nal_unit) => { + self.current_mut(packet.timestamp)?.nal_units.push(nal_unit.to_vec()); + } + H264Payload::Aggregation(nal_units) => { + self.push_aggregation(packet.timestamp, nal_units)?; + } + H264Payload::Fragment(fragment) => self.apply_fragment(packet.timestamp, fragment)?, } - Ok(()) } @@ -45,117 +153,55 @@ impl RtpAccessUnitAssembler { &mut self, packet: &RtpPacket<'_>, ) -> Result<(), RtpDepacketizerError> { - let payload = packet.payload; - if payload.len() < 2 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - let nal_type = (payload[0] >> 1) & 0x3f; - - match nal_type { - 0..=47 => self.current_mut(packet.timestamp)?.nal_units.push(payload.to_vec()), - 48 => self.push_h26x_aggregation(packet.timestamp, &payload[2..])?, - 49 => self.push_h265_fragment(packet.timestamp, payload)?, - _ => return Err(RtpDepacketizerError::UnsupportedPayload), + match parse_h265_payload(packet.payload)? { + H265Payload::Nal(nal_unit) => { + self.current_mut(packet.timestamp)?.nal_units.push(nal_unit.to_vec()); + } + H265Payload::Aggregation(nal_units) => { + self.push_aggregation(packet.timestamp, nal_units)?; + } + H265Payload::Fragment(fragment) => self.apply_fragment(packet.timestamp, fragment)?, } - Ok(()) } - /// Unpacks the length-prefixed NAL units of an H.264 STAP-A or H.265 AP - /// payload, whose aggregation headers the caller has already stripped. - /// - /// The layout is the same 2-byte-length-prefixed sequence as an - /// AVC-format access unit, so the AVC splitter does the walking. - fn push_h26x_aggregation( + /// Adds the complete NAL units of an aggregation packet. + fn push_aggregation( &mut self, rtp_timestamp: u32, - payload: &[u8], + nal_units: Vec<&[u8]>, ) -> Result<(), RtpDepacketizerError> { - let nal_units: Vec> = - avc_nalus(payload, 2)?.into_iter().map(<[u8]>::to_vec).collect(); + let nal_units: Vec> = nal_units.into_iter().map(<[u8]>::to_vec).collect(); self.current_mut(rtp_timestamp)?.nal_units.extend(nal_units); Ok(()) } - fn push_h264_fu_a( + /// Applies one NAL fragment: starts, extends, or completes the pending + /// fragmented NAL unit. + fn apply_fragment( &mut self, rtp_timestamp: u32, - payload: &[u8], + fragment: Fragment<'_>, ) -> Result<(), RtpDepacketizerError> { - if payload.len() < 2 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - - let indicator = payload[0]; - let header = payload[1]; - let start = (header & 0x80) != 0; - let end = (header & 0x40) != 0; - let nal_type = header & 0x1f; - if nal_type == 0 || nal_type > 23 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - - if start { - let mut nal_unit = Vec::with_capacity(1 + payload.len().saturating_sub(2)); - nal_unit.push((indicator & 0xe0) | nal_type); - nal_unit.extend_from_slice(&payload[2..]); + if fragment.start { + let nal_header = fragment.nal_header.as_slice(); + let mut nal_unit = Vec::with_capacity(nal_header.len() + fragment.payload.len()); + nal_unit.extend_from_slice(nal_header); + nal_unit.extend_from_slice(fragment.payload); self.fragment = Some(FragmentState { rtp_timestamp, nal_unit }); return Ok(()); } - let Some(fragment) = - self.fragment.as_mut().filter(|fragment| fragment.rtp_timestamp == rtp_timestamp) + let Some(state) = + self.fragment.as_mut().filter(|state| state.rtp_timestamp == rtp_timestamp) else { // A continuation without its start means the preceding packets were lost. self.discard_in_progress(); return Ok(()); }; - fragment.nal_unit.extend_from_slice(&payload[2..]); + state.nal_unit.extend_from_slice(fragment.payload); - if end { - let nal_unit = - self.fragment.take().ok_or(RtpDepacketizerError::InvalidFragment)?.nal_unit; - self.current_mut(rtp_timestamp)?.nal_units.push(nal_unit); - } - Ok(()) - } - - fn push_h265_fragment( - &mut self, - rtp_timestamp: u32, - payload: &[u8], - ) -> Result<(), RtpDepacketizerError> { - if payload.len() < 3 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - - let fu_header = payload[2]; - let start = (fu_header & 0x80) != 0; - let end = (fu_header & 0x40) != 0; - let nal_type = fu_header & 0x3f; - if nal_type > 47 { - return Err(RtpDepacketizerError::UnsupportedPayload); - } - - if start { - let mut nal_unit = Vec::with_capacity(2 + payload.len().saturating_sub(3)); - nal_unit.push((payload[0] & 0x81) | (nal_type << 1)); - nal_unit.push(payload[1]); - nal_unit.extend_from_slice(&payload[3..]); - self.fragment = Some(FragmentState { rtp_timestamp, nal_unit }); - return Ok(()); - } - - let Some(fragment) = - self.fragment.as_mut().filter(|fragment| fragment.rtp_timestamp == rtp_timestamp) - else { - // A continuation without its start means the preceding packets were lost. - self.discard_in_progress(); - return Ok(()); - }; - fragment.nal_unit.extend_from_slice(&payload[3..]); - - if end { + if fragment.end { let nal_unit = self.fragment.take().ok_or(RtpDepacketizerError::InvalidFragment)?.nal_unit; self.current_mut(rtp_timestamp)?.nal_units.push(nal_unit); @@ -277,6 +323,68 @@ mod tests { crate::encoded::h26x::annex_b_nalus(&access_unit.payload) } + #[test] + fn parses_h264_payloads() { + assert_eq!(parse_h264_payload(&[0x65, 1, 2]), Ok(H264Payload::Nal(&[0x65, 1, 2]))); + assert_eq!( + parse_h264_payload(&[0x18, 0, 2, 0x67, 9, 0, 1, 0x68]), + Ok(H264Payload::Aggregation(vec![&[0x67, 9][..], &[0x68][..]])) + ); + // FU-A start: the NAL header is rebuilt from the indicator's NRI + // bits and the FU header's type. + assert_eq!( + parse_h264_payload(&[0x7c, 0x85, 1, 2]), + Ok(H264Payload::Fragment(Fragment { + start: true, + end: false, + nal_header: NalHeader::H264(0x65), + payload: &[1, 2], + })) + ); + // FU-A end. + assert_eq!( + parse_h264_payload(&[0x7c, 0x45, 3]), + Ok(H264Payload::Fragment(Fragment { + start: false, + end: true, + nal_header: NalHeader::H264(0x65), + payload: &[3], + })) + ); + // Truncated, forbidden fragment types, and unsupported packet types. + assert!(parse_h264_payload(&[]).is_err()); + assert!(parse_h264_payload(&[0x7c]).is_err()); + assert!(parse_h264_payload(&[0x7c, 0x80]).is_err()); // FU of type 0 + assert!(parse_h264_payload(&[0x19, 1]).is_err()); // STAP-B + } + + #[test] + fn parses_h265_payloads() { + assert_eq!( + parse_h265_payload(&[0x26, 0x01, 1]), + Ok(H265Payload::Nal(&[0x26, 0x01, 1])) + ); + assert_eq!( + parse_h265_payload(&[0x60, 0x01, 0, 2, 0x40, 0x01]), + Ok(H265Payload::Aggregation(vec![&[0x40, 0x01][..]])) + ); + // FU start: the 2-byte NAL header keeps the layer and temporal-id + // bits and takes the type from the FU header. + assert_eq!( + parse_h265_payload(&[0x62, 0x01, 0x93, 1, 2]), + Ok(H265Payload::Fragment(Fragment { + start: true, + end: false, + nal_header: NalHeader::H265([0x26, 0x01]), + payload: &[1, 2], + })) + ); + // Truncated inputs and reserved fragment types. + assert!(parse_h265_payload(&[0x62]).is_err()); + assert!(parse_h265_payload(&[0x62, 0x01]).is_err()); + assert!(parse_h265_payload(&[0x62, 0x01, 0xb0]).is_err()); // FU of type 48 + } + #[test] fn assembles_h264_single_nal_access_unit() { let mut assembler = assembler(EncodedVideoCodec::H264); From e8ffff743f1809efdb74d869410068c3f4dee979 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:33:16 -0700 Subject: [PATCH 31/38] Use byte reader in dimension parsing --- livekit-capture/src/sources/rtsp/bits.rs | 30 ++----------------- .../src/sources/rtsp/dimensions.rs | 20 ++++++------- 2 files changed, 13 insertions(+), 37 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/bits.rs b/livekit-capture/src/sources/rtsp/bits.rs index c70182b4e..04f24efd6 100644 --- a/livekit-capture/src/sources/rtsp/bits.rs +++ b/livekit-capture/src/sources/rtsp/bits.rs @@ -157,24 +157,6 @@ impl<'a> ByteReader<'a> { } } -/// Reads an AV1/LEB128 length from `bytes` at `cursor`, advancing it. -pub(super) fn read_leb128(bytes: &[u8], cursor: &mut usize) -> Option { - let mut value = 0usize; - let mut shift = 0usize; - loop { - let &byte = bytes.get(*cursor)?; - *cursor += 1; - value |= usize::from(byte & 0x7f) << shift; - if byte & 0x80 == 0 { - return Some(value); - } - shift += 7; - if shift >= usize::BITS as usize { - return None; - } - } -} - /// Appends `value` to `out` as LEB128. pub(super) fn write_leb128(mut value: usize, out: &mut Vec) { loop { @@ -247,15 +229,9 @@ mod tests { for value in [0usize, 1, 127, 128, 300, 16_383, 16_384, usize::from(u16::MAX)] { let mut encoded = Vec::new(); write_leb128(value, &mut encoded); - let mut cursor = 0; - assert_eq!(read_leb128(&encoded, &mut cursor), Some(value)); - assert_eq!(cursor, encoded.len()); + let mut reader = ByteReader::new(&encoded); + assert_eq!(reader.get_leb128(), Some(value)); + assert!(reader.is_empty()); } } - - #[test] - fn leb128_rejects_truncated_input() { - let mut cursor = 0; - assert_eq!(read_leb128(&[0x80], &mut cursor), None); - } } diff --git a/livekit-capture/src/sources/rtsp/dimensions.rs b/livekit-capture/src/sources/rtsp/dimensions.rs index 6ccf712c4..328a38623 100644 --- a/livekit-capture/src/sources/rtsp/dimensions.rs +++ b/livekit-capture/src/sources/rtsp/dimensions.rs @@ -17,7 +17,7 @@ //! Every parser returns `None` on malformed or unexpected input; discovery //! treats that as "not discoverable" rather than a stream error. -use super::bits::{read_leb128, BitReader}; +use super::bits::{BitReader, ByteReader}; use crate::{ encoded::{ h26x::{annex_b_nalus, h264_nal_type, h265_nal_type}, @@ -302,29 +302,29 @@ fn vp9_keyframe_resolution(payload: &[u8]) -> Option { /// Parses the maximum frame dimensions from the sequence header OBU of an /// AV1 access unit built from size-prefixed OBUs. fn av1_sequence_header_resolution(payload: &[u8]) -> Option { - let mut cursor = 0; - while cursor < payload.len() { - let header = *payload.get(cursor)?; + let mut reader = ByteReader::new(payload); + while !reader.is_empty() { + let header = reader.get_u8()?; if header & 0x80 != 0 { return None; // obu_forbidden_bit } let obu_type = (header & 0x78) >> 3; let has_extension = header & 0x04 != 0; let has_size = header & 0x02 != 0; - cursor += if has_extension { 2 } else { 1 }; + if has_extension { + reader.skip(1)?; + } if !has_size { // Without a size field the OBU extends to the end of the unit. return (obu_type == 1) - .then(|| av1_sequence_header_obu_resolution(payload.get(cursor..)?)) + .then(|| av1_sequence_header_obu_resolution(reader.take_rest())) .flatten(); } - let size = read_leb128(payload, &mut cursor)?; - let end = cursor.checked_add(size)?; - let obu_payload = payload.get(cursor..end)?; + let size = reader.get_leb128()?; + let obu_payload = reader.take(size)?; if obu_type == 1 { return av1_sequence_header_obu_resolution(obu_payload); } - cursor = end; } None } From 4f470123f8d76fa60a315aa2a8b3be13de5ec548 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:34:21 -0700 Subject: [PATCH 32/38] Name NAL unit types --- livekit-capture/src/encoded/h26x.rs | 26 ++++++++++++++----- .../src/sources/rtsp/dimensions.rs | 6 ++--- livekit-capture/src/sources/rtsp/rtp/h26x.rs | 20 ++++++++------ 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 982eee423..af5e995f8 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -454,9 +454,9 @@ fn is_keyframe_nalus( nal_units: &[&[u8]], ) -> Result { match codec { - EncodedVideoCodec::H264 => { - nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) - } + EncodedVideoCodec::H264 => nal_units + .iter() + .try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == H264_NAL_IDR)), EncodedVideoCodec::H265 => { let mut has_vps = false; let mut has_sps = false; @@ -465,10 +465,10 @@ fn is_keyframe_nalus( for nal in nal_units { match h265_nal_type(nal)? { - 32 => has_vps = true, - 33 => has_sps = true, - 34 => has_pps = true, - 19 | 20 => has_idr = true, + H265_NAL_VPS => has_vps = true, + H265_NAL_SPS => has_sps = true, + H265_NAL_PPS => has_pps = true, + H265_NAL_IDR_W_RADL | H265_NAL_IDR_N_LP => has_idr = true, _ => {} } } @@ -481,6 +481,18 @@ fn is_keyframe_nalus( } } +/// H.264 NAL unit types (ITU-T H.264 table 7-1). +pub(crate) const H264_NAL_IDR: u8 = 5; +pub(crate) const H264_NAL_SPS: u8 = 7; +pub(crate) const H264_NAL_PPS: u8 = 8; + +/// H.265 NAL unit types (ITU-T H.265 table 7-1). +pub(crate) const H265_NAL_IDR_W_RADL: u8 = 19; +pub(crate) const H265_NAL_IDR_N_LP: u8 = 20; +pub(crate) const H265_NAL_VPS: u8 = 32; +pub(crate) const H265_NAL_SPS: u8 = 33; +pub(crate) const H265_NAL_PPS: u8 = 34; + /// Extracts the type of an H.264 NAL unit from its header. pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { let header = nal.first().ok_or(H26xParseError::EmptyPayload)?; diff --git a/livekit-capture/src/sources/rtsp/dimensions.rs b/livekit-capture/src/sources/rtsp/dimensions.rs index 328a38623..b39ce98fe 100644 --- a/livekit-capture/src/sources/rtsp/dimensions.rs +++ b/livekit-capture/src/sources/rtsp/dimensions.rs @@ -20,7 +20,7 @@ use super::bits::{BitReader, ByteReader}; use crate::{ encoded::{ - h26x::{annex_b_nalus, h264_nal_type, h265_nal_type}, + h26x::{annex_b_nalus, h264_nal_type, h265_nal_type, H264_NAL_SPS, H265_NAL_SPS}, EncodedVideoCodec, }, primitive::VideoResolution, @@ -34,11 +34,11 @@ pub(super) fn access_unit_resolution( match codec { EncodedVideoCodec::H264 => annex_b_nalus(payload) .into_iter() - .find(|nal| matches!(h264_nal_type(nal), Ok(7))) + .find(|nal| matches!(h264_nal_type(nal), Ok(H264_NAL_SPS))) .and_then(|nal| sps_resolution(codec, nal)), EncodedVideoCodec::H265 => annex_b_nalus(payload) .into_iter() - .find(|nal| matches!(h265_nal_type(nal), Ok(33))) + .find(|nal| matches!(h265_nal_type(nal), Ok(H265_NAL_SPS))) .and_then(|nal| sps_resolution(codec, nal)), EncodedVideoCodec::VP8 => vp8_keyframe_resolution(payload), EncodedVideoCodec::VP9 => vp9_keyframe_resolution(payload), diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs index 4a85c5c38..894b0adc9 100644 --- a/livekit-capture/src/sources/rtsp/rtp/h26x.rs +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -17,7 +17,11 @@ use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; use crate::{ encoded::{ - h26x::{access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type}, + h26x::{ + access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type, H264_NAL_IDR, + H264_NAL_PPS, H264_NAL_SPS, H265_NAL_IDR_N_LP, H265_NAL_IDR_W_RADL, H265_NAL_PPS, + H265_NAL_SPS, H265_NAL_VPS, + }, EncodedFrameType, EncodedVideoCodec, }, sources::rtsp::bits::ByteReader, @@ -279,16 +283,16 @@ impl NalPresence { fn record(&mut self, codec: EncodedVideoCodec, nal: &[u8]) { match codec { EncodedVideoCodec::H264 => match h264_nal_type(nal) { - Ok(5) => self.idr = true, - Ok(7) => self.sps = true, - Ok(8) => self.pps = true, + Ok(H264_NAL_IDR) => self.idr = true, + Ok(H264_NAL_SPS) => self.sps = true, + Ok(H264_NAL_PPS) => self.pps = true, _ => {} }, EncodedVideoCodec::H265 => match h265_nal_type(nal) { - Ok(19 | 20) => self.idr = true, - Ok(32) => self.vps = true, - Ok(33) => self.sps = true, - Ok(34) => self.pps = true, + Ok(H265_NAL_IDR_W_RADL | H265_NAL_IDR_N_LP) => self.idr = true, + Ok(H265_NAL_VPS) => self.vps = true, + Ok(H265_NAL_SPS) => self.sps = true, + Ok(H265_NAL_PPS) => self.pps = true, _ => {} }, _ => {} From 7fbc7d567b70211460871d7d100e9fce3c661bf4 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:36:21 -0700 Subject: [PATCH 33/38] Escape server strings in errors --- livekit-capture/src/sources/rtsp/auth.rs | 19 +++++++++++++++++-- livekit-capture/src/sources/rtsp/client.rs | 10 ++++++---- livekit-capture/src/sources/rtsp/mod.rs | 6 ++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/auth.rs b/livekit-capture/src/sources/rtsp/auth.rs index 127ed1a4b..5e9e6556c 100644 --- a/livekit-capture/src/sources/rtsp/auth.rs +++ b/livekit-capture/src/sources/rtsp/auth.rs @@ -70,7 +70,7 @@ impl RtspAuthContext { // RTSP requests carry no body relevant to `auth-int`. body: Some(&[]), }) - .map_err(RtspVideoSourceError::Auth)?; + .map_err(|err| RtspVideoSourceError::Auth(super::sanitized(err)))?; Ok(Some(authorization)) } @@ -87,7 +87,9 @@ impl RtspAuthContext { for challenges in response.headers("www-authenticate") { builder = builder.challenges(challenges); } - self.client = Some(builder.build().map_err(RtspVideoSourceError::Auth)?); + // Build errors can quote the server's challenge bytes. + self.client = + Some(builder.build().map_err(|err| RtspVideoSourceError::Auth(super::sanitized(err)))?); Ok(()) } } @@ -159,6 +161,19 @@ mod tests { assert!(matches!(err, RtspVideoSourceError::Auth(_)), "unexpected error: {err:?}"); } + #[test] + fn escapes_control_characters_in_auth_errors() { + let mut context = context_with_credentials(); + let err = context + .update_from_unauthorized(&unauthorized(&["Bearer \u{7f}\u{1b}[31mfake"])) + .unwrap_err(); + let message = err.to_string(); + assert!( + !message.chars().any(char::is_control), + "control characters in message: {message:?}" + ); + } + #[test] fn requires_credentials_for_challenges() { let mut context = RtspAuthContext::new(None); diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs index 71022febb..9eddbc4c1 100644 --- a/livekit-capture/src/sources/rtsp/client.rs +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -26,6 +26,8 @@ use std::{ use bytes::{Buf, Bytes, BytesMut}; use rtsp_types::{headers, HeaderName, Message, Method, ParseError, Url, Version}; +#[cfg(feature = "source-rtsp-tls")] +use super::sanitized; use super::{auth::RtspAuthContext, auth::RtspCredentials, RtspPhase, RtspVideoSourceError}; /// Socket read timeout: the poll granularity for the stop token, keepalives, @@ -547,7 +549,7 @@ mod tls { use rustls::{ClientConfig, ClientConnection, RootCertStore, StreamOwned}; use rustls_pki_types::ServerName; - use super::{is_timeout_io_error, RtspPhase, RtspVideoSourceError}; + use super::{is_timeout_io_error, sanitized, RtspPhase, RtspVideoSourceError}; /// Establishes TLS over a connected TCP stream, driving the handshake to /// completion bounded by `deadline`. @@ -565,9 +567,9 @@ mod tls { // `ServerName` accepts both DNS names and the IP literals cameras // are usually addressed by. let server_name = ServerName::try_from(host.to_owned()) - .map_err(|err| RtspVideoSourceError::Tls(err.to_string()))?; + .map_err(|err| RtspVideoSourceError::Tls(sanitized(err.to_string())))?; let mut connection = ClientConnection::new(Arc::new(config), server_name) - .map_err(|err| RtspVideoSourceError::Tls(err.to_string()))?; + .map_err(|err| RtspVideoSourceError::Tls(sanitized(err.to_string())))?; let mut stream = stream; while connection.is_handshaking() { @@ -581,7 +583,7 @@ mod tls { } // rustls reports TLS-level handshake failures as InvalidData. Err(err) if err.kind() == io::ErrorKind::InvalidData => { - return Err(RtspVideoSourceError::Tls(err.to_string())); + return Err(RtspVideoSourceError::Tls(sanitized(err.to_string()))); } Err(err) => return Err(err.into()), } diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index 65c06edd4..e10f07883 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -613,6 +613,12 @@ fn duration_ms(ms: Option, default: Duration) -> Duration { ms.map(|ms| Duration::from_millis(ms.into())).unwrap_or(default) } +/// Escapes text that may quote server-provided bytes before it enters an +/// error message or a log line. +fn sanitized(text: impl AsRef) -> String { + text.as_ref().escape_debug().to_string() +} + #[cfg(test)] mod tests { use std::{ From 39b5dcdbef2a685788a53d4791fee3fd2e75dc74 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:45:46 -0700 Subject: [PATCH 34/38] Expand integration tests --- livekit-capture/tests/README.md | 5 +- livekit-capture/tests/common/rtsp.rs | 146 +++++++++++----- livekit-capture/tests/source_rtsp_test.rs | 201 ++++++++++++++++++++-- 3 files changed, 298 insertions(+), 54 deletions(-) diff --git a/livekit-capture/tests/README.md b/livekit-capture/tests/README.md index 397fddd5e..a0ba4b40e 100644 --- a/livekit-capture/tests/README.md +++ b/livekit-capture/tests/README.md @@ -15,8 +15,9 @@ section below documents its prerequisites. Each test starts an in-process GStreamer RTSP server on an ephemeral localhost port and streams real encoded video to the source. The system -GStreamer installation must include the RTSP server library and the x264, -x265, and VP8 encoder plugins: +GStreamer installation must include the RTSP server library, the x264, +x265, VP8/VP9, and AV1 encoder plugins, and the RTP payloaders from +gst-plugins-rs (`rtpav1pay`): - macOS: `brew install gstreamer` - Debian/Ubuntu: `apt install libgstrtspserver-1.0-dev diff --git a/livekit-capture/tests/common/rtsp.rs b/livekit-capture/tests/common/rtsp.rs index 65b414f68..5204b73b5 100644 --- a/livekit-capture/tests/common/rtsp.rs +++ b/livekit-capture/tests/common/rtsp.rs @@ -22,29 +22,69 @@ use std::{ use gstreamer::{self as gst, glib}; use gstreamer_rtsp_server::{prelude::*, RTSPAuth, RTSPMediaFactory, RTSPServer, RTSPToken}; -use livekit_capture::sources::rtsp::RtspVideoSourceConfig; - -/// Test streams are 640x480 at 30 fps with a keyframe every 30 frames. -pub const TEST_WIDTH: u32 = 640; -pub const TEST_HEIGHT: u32 = 480; - -pub const H264_PIPELINE: &str = "videotestsrc is-live=true \ - ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ - ! x264enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ - byte-stream=true aud=true \ - ! h264parse config-interval=-1 ! rtph264pay name=pay0 pt=96 config-interval=1"; - -pub const H265_PIPELINE: &str = "videotestsrc is-live=true \ - ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ - ! x265enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ - option-string=repeat-headers=1:aud=1:open-gop=0 \ - ! h265parse config-interval=-1 ! rtph265pay name=pay0 pt=96 config-interval=1"; - -pub const VP8_PIPELINE: &str = "videotestsrc is-live=true \ - ! video/x-raw,width=640,height=480,framerate=30/1 ! videoconvert \ - ! vp8enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ - target-bitrate=500000 \ - ! rtpvp8pay name=pay0 pt=96"; +use livekit_capture::{primitive::VideoResolution, sources::rtsp::RtspVideoSourceConfig}; + +/// Default test streams are 640x480 at 30 fps with a keyframe every 30 +/// frames. +pub const TEST_RESOLUTION: VideoResolution = VideoResolution::new(640, 480); + +/// Codecs the test server can encode. +#[derive(Debug, Clone, Copy)] +pub enum TestCodec { + H264, + H265, + Vp8, + Vp9, + Av1, +} + +/// Builds an encoder pipeline for the test server at the given resolution. +pub fn pipeline(codec: TestCodec, resolution: VideoResolution) -> String { + let encode = match codec { + TestCodec::H264 => { + "x264enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + byte-stream=true aud=true \ + ! h264parse config-interval=-1 ! rtph264pay name=pay0 pt=96 config-interval=1" + } + TestCodec::H265 => { + "x265enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + option-string=repeat-headers=1:aud=1:open-gop=0 \ + ! h265parse config-interval=-1 ! rtph265pay name=pay0 pt=96 config-interval=1" + } + TestCodec::Vp8 => { + "vp8enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500000 ! rtpvp8pay name=pay0 pt=96" + } + TestCodec::Vp9 => { + "vp9enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500000 ! rtpvp9pay name=pay0 pt=96" + } + TestCodec::Av1 => { + "av1enc cpu-used=8 usage-profile=realtime keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500 ! av1parse \ + ! video/x-av1,stream-format=obu-stream,alignment=tu ! rtpav1pay name=pay0 pt=96" + } + }; + format!( + "videotestsrc is-live=true \ + ! video/x-raw,width={},height={},framerate=30/1 ! videoconvert ! {encode}", + resolution.width, resolution.height, + ) +} + +/// Builds a pipeline at the default resolution. +pub fn default_pipeline(codec: TestCodec) -> String { + pipeline(codec, TEST_RESOLUTION) +} + +/// Builds a two-track pipeline: video at the default resolution plus a PCMA +/// audio track, mirroring the SDP shape of a typical camera. +pub fn default_pipeline_with_audio(codec: TestCodec) -> String { + format!( + "{} audiotestsrc is-live=true ! alawenc ! rtppcmapay name=pay1 pt=8", + default_pipeline(codec), + ) +} /// An in-process GStreamer RTSP server serving one launch pipeline at /// `/test` on an ephemeral localhost port. @@ -55,11 +95,28 @@ pub struct RtspTestServer { tls: bool, } +/// Authentication required by a test server. +#[derive(Debug, Clone, Copy)] +enum TestAuth<'a> { + None, + Basic { username: &'a str, password: &'a str }, + Digest { username: &'a str, password: &'a str }, +} + impl RtspTestServer { - /// Starts a server streaming `media_pipeline`, which must end in an RTP + /// Starts a server streaming `media_pipeline`, which must contain an RTP /// payloader named `pay0`. pub fn launch(media_pipeline: &str) -> Self { - Self::launch_inner(media_pipeline, None, false) + Self::launch_inner(media_pipeline, TestAuth::None, false) + } + + /// Starts a server that requires Basic authentication. + pub fn launch_with_basic_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, TestAuth::Basic { username, password }, false) } /// Starts a server that requires Digest authentication. @@ -68,13 +125,13 @@ impl RtspTestServer { username: &str, password: &str, ) -> Self { - Self::launch_inner(media_pipeline, Some((username, password)), false) + Self::launch_inner(media_pipeline, TestAuth::Digest { username, password }, false) } /// Starts a server that requires TLS (`rtsps://`), presenting a /// freshly generated self-signed certificate. pub fn launch_tls(media_pipeline: &str) -> Self { - Self::launch_inner(media_pipeline, None, true) + Self::launch_inner(media_pipeline, TestAuth::None, true) } /// Starts a server that requires both TLS and Digest authentication. @@ -83,10 +140,10 @@ impl RtspTestServer { username: &str, password: &str, ) -> Self { - Self::launch_inner(media_pipeline, Some((username, password)), true) + Self::launch_inner(media_pipeline, TestAuth::Digest { username, password }, true) } - fn launch_inner(media_pipeline: &str, digest: Option<(&str, &str)>, tls: bool) -> Self { + fn launch_inner(media_pipeline: &str, test_auth: TestAuth<'_>, tls: bool) -> Self { gst::init().expect("failed to initialize GStreamer"); // Each server runs on its own main context so parallel tests never @@ -101,7 +158,7 @@ impl RtspTestServer { factory.set_launch(&format!("( {media_pipeline} )")); factory.set_shared(false); - if digest.is_some() || tls { + if !matches!(test_auth, TestAuth::None) || tls { factory.add_role_from_structure( &gst::Structure::builder("user") .field("media.factory.access", true) @@ -114,14 +171,21 @@ impl RtspTestServer { // every connection to this server instance. auth.set_tls_certificate(Some(&self_signed_certificate())); } - if let Some((username, password)) = digest { - let token = RTSPToken::builder().field("media.factory.role", "user").build(); - auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); - auth.add_digest(username, password, &token); - } else { - // TLS without authentication: admit anonymous clients. - let mut token = RTSPToken::builder().field("media.factory.role", "user").build(); - auth.set_default_token(Some(&mut token)); + let token = RTSPToken::builder().field("media.factory.role", "user").build(); + match test_auth { + TestAuth::None => { + // TLS without authentication: admit anonymous clients. + let mut token = token; + auth.set_default_token(Some(&mut token)); + } + TestAuth::Basic { username, password } => { + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Basic); + auth.add_basic(&RTSPAuth::make_basic(username, password), &token); + } + TestAuth::Digest { username, password } => { + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); + auth.add_digest(username, password, &token); + } } server.set_auth(Some(&auth)); } @@ -136,7 +200,11 @@ impl RtspTestServer { log::info!( "RTSP test server listening at {}://127.0.0.1:{port}/test{}", if tls { "rtsps" } else { "rtsp" }, - if digest.is_some() { " (digest auth)" } else { "" }, + match test_auth { + TestAuth::None => "", + TestAuth::Basic { .. } => " (basic auth)", + TestAuth::Digest { .. } => " (digest auth)", + }, ); log::debug!("RTSP test server pipeline: {media_pipeline}"); diff --git a/livekit-capture/tests/source_rtsp_test.rs b/livekit-capture/tests/source_rtsp_test.rs index 1406da9f3..4692ca7a8 100644 --- a/livekit-capture/tests/source_rtsp_test.rs +++ b/livekit-capture/tests/source_rtsp_test.rs @@ -26,8 +26,8 @@ use test_log::test; use common::{ pull_access_units, rtsp::{ - test_config, RtspTestServer, H264_PIPELINE, H265_PIPELINE, TEST_HEIGHT, TEST_WIDTH, - VP8_PIPELINE, + default_pipeline, default_pipeline_with_audio, pipeline, test_config, RtspTestServer, + TestCodec, TEST_RESOLUTION, }, }; use livekit_capture::{ @@ -36,8 +36,6 @@ use livekit_capture::{ sources::rtsp::{RtspVideoSource, RtspVideoSourceConfig}, }; -const TEST_RESOLUTION: VideoResolution = VideoResolution::new(TEST_WIDTH, TEST_HEIGHT); - fn h264_nal_types(payload: &[u8]) -> Vec { annex_b_nalus(payload).iter().map(|nal| nal[0] & 0x1f).collect() } @@ -59,7 +57,7 @@ fn assert_increasing_timestamps(access_units: &[livekit_capture::encoded::OwnedE #[test] fn streams_h264_access_units() { - let server = RtspTestServer::launch(H264_PIPELINE); + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { codec: Some(EncodedVideoCodec::H264), resolution: Some(TEST_RESOLUTION), @@ -90,7 +88,7 @@ fn streams_h264_access_units() { #[test] fn discovers_h264_resolution() { - let server = RtspTestServer::launch(H264_PIPELINE); + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); let mut source = RtspVideoSource::new_blocking(test_config(server.url())).expect("failed to connect"); @@ -104,7 +102,7 @@ fn discovers_h264_resolution() { #[test] fn streams_h265_access_units() { - let server = RtspTestServer::launch(H265_PIPELINE); + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H265)); let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { codec: Some(EncodedVideoCodec::H265), ..test_config(server.url()) @@ -131,7 +129,7 @@ fn streams_h265_access_units() { #[test] fn streams_vp8_access_units() { - let server = RtspTestServer::launch(VP8_PIPELINE); + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Vp8)); let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { codec: Some(EncodedVideoCodec::VP8), ..test_config(server.url()) @@ -152,7 +150,7 @@ fn streams_vp8_access_units() { #[test] fn rejects_codec_mismatch() { - let server = RtspTestServer::launch(H264_PIPELINE); + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { codec: Some(EncodedVideoCodec::VP8), resolution: Some(TEST_RESOLUTION), @@ -165,7 +163,7 @@ fn rejects_codec_mismatch() { #[test] fn streams_h264_over_rtsps() { - let server = RtspTestServer::launch_tls(H264_PIPELINE); + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H264)); let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { codec: Some(EncodedVideoCodec::H264), resolution: Some(TEST_RESOLUTION), @@ -182,7 +180,7 @@ fn streams_h264_over_rtsps() { #[test] fn rejects_untrusted_tls_certificate() { - let server = RtspTestServer::launch_tls(H264_PIPELINE); + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H264)); // Default configuration verifies against the system roots, which must // reject the server's self-signed certificate. let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { @@ -200,7 +198,7 @@ fn rejects_untrusted_tls_certificate() { #[test] fn authenticates_with_digest_over_rtsps() { - let server = RtspTestServer::launch_tls_with_digest_auth(H264_PIPELINE, "admin", "secret"); + let server = RtspTestServer::launch_tls_with_digest_auth(&default_pipeline(TestCodec::H264), "admin", "secret"); let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { username: Some("admin".to_owned()), password: Some("secret".to_owned()), @@ -217,7 +215,7 @@ fn authenticates_with_digest_over_rtsps() { #[test] fn authenticates_with_digest() { - let server = RtspTestServer::launch_with_digest_auth(H264_PIPELINE, "admin", "secret"); + let server = RtspTestServer::launch_with_digest_auth(&default_pipeline(TestCodec::H264), "admin", "secret"); // Without credentials the server's challenge cannot be answered. let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { @@ -239,3 +237,180 @@ fn authenticates_with_digest() { let first = pull_access_units(&mut source, 1).remove(0); assert_eq!(first.frame_type, EncodedFrameType::Key); } + +#[test] +fn selects_video_track_among_audio() { + let server = RtspTestServer::launch(&default_pipeline_with_audio(TestCodec::H264)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // The video track is selected from the two-track SDP; the audio track + // is neither set up nor streamed. + assert_eq!(source.codec(), EncodedVideoCodec::H264); + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + } +} + +#[test] +fn discovers_cropped_h264_resolution() { + // 1080p is coded as 1088 rows plus SPS frame cropping; discovery must + // report the display resolution from a real encoder's SPS. + let server = RtspTestServer::launch(&pipeline(TestCodec::H264, VideoResolution::new(1920, 1080))); + let source = + RtspVideoSource::new_blocking(test_config(server.url())).expect("failed to connect"); + + assert_eq!(source.resolution(), VideoResolution::new(1920, 1080)); +} + +#[test] +fn streams_vp9_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Vp9)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP9), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // Discovery parses the first keyframe's uncompressed header. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn streams_av1_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Av1)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::AV1), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // Discovery parses the sequence header OBU of the first keyframe. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::AV1); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn rejects_wrong_credentials() { + let server = RtspTestServer::launch_with_digest_auth( + &default_pipeline(TestCodec::H264), + "admin", + "secret", + ); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("wrong".to_owned()), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + // One authenticated retry, then a clean failure — no retry loop. + assert!(err.to_string().contains("401"), "unexpected error: {err}"); +} + +#[test] +fn rejects_unknown_path() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url().replace("/test", "/wrong")) + }) + .unwrap_err(); + + assert!(err.to_string().contains("404"), "unexpected error: {err}"); +} + +#[test] +fn rejects_mismatched_declared_resolution() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + // A declared resolution skips discovery and is verified against the + // first keyframe instead. + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(VideoResolution::new(1280, 720)), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + let stop = livekit_capture::pump::PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("1280x720"), "unexpected error: {err}"); +} + +#[test] +fn reconnects_after_drop() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + // The previous source's teardown must leave the server usable for a + // fresh connection. + for _ in 0..2 { + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); + } +} + +#[test] +fn authenticates_with_basic() { + let server = RtspTestServer::launch_with_basic_auth( + &default_pipeline(TestCodec::H264), + "admin", + "secret", + ); + + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + assert!(err.to_string().contains("credentials"), "unexpected error: {err}"); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect with credentials"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} + +#[test] +fn streams_h265_over_rtsps() { + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H265)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H265), + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect over TLS"); + + assert_eq!(source.resolution(), TEST_RESOLUTION); + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} From ad985d0f8c99928e57b55d4639eb16434ce700b4 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 16:57:26 -0700 Subject: [PATCH 35/38] Log additional video tracks --- livekit-capture/src/sources/rtsp/mod.rs | 5 +++++ livekit-capture/src/sources/rtsp/sdp.rs | 23 +++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index e10f07883..2b82b4d2f 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -27,6 +27,11 @@ //! The connection is not re-established on failure: a connection error ends //! the source with an error, and a clean server-side end of stream ends it //! like any finite source. +//! +//! One video track is ingested per source: when the SDP offers several, the +//! first one carrying a supported (or the configured) codec is used and the +//! choice is logged. Cameras exposing multiple streams as separate URLs are +//! served by one source per URL. mod auth; mod bits; diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs index 6742d199a..ca809d0b1 100644 --- a/livekit-capture/src/sources/rtsp/sdp.rs +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -74,6 +74,29 @@ pub(super) fn parse_sdp_session( })?; let session_control = attribute_value(&session.attributes, "control"); + // Selecting among multiple video tracks is not supported (only ordinal + // selection would have standard SDP footing); surface the choice so it + // is not hidden. + let video_tracks = || session.medias.iter().filter(|media| media.media == "video"); + let video_track_count = video_tracks().count(); + if video_track_count > 1 { + let summary = video_tracks() + .map(|media| { + let codecs: Vec<&str> = attribute_values(&media.attributes, "rtpmap") + .filter_map(|rtpmap| rtpmap.split_whitespace().nth(1)) + .filter_map(|encoding| encoding.split('/').next()) + .collect(); + if codecs.is_empty() { "?".to_owned() } else { codecs.join("+") } + }) + .collect::>() + .join(", "); + log::info!( + "RTSP SDP offers {video_track_count} video tracks ({}); \ + using the first one with a supported codec", + super::sanitized(summary), + ); + } + let mut offered = Vec::new(); for media in &session.medias { if media.media != "video" { From fd68a06de7f5c6ec4bdcb5607612883131041e46 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 26 Aug 2026 17:08:31 -0700 Subject: [PATCH 36/38] Document best practices for untrusted input --- livekit-capture/AGENTS.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md index 745de9053..af0927821 100644 --- a/livekit-capture/AGENTS.md +++ b/livekit-capture/AGENTS.md @@ -42,3 +42,31 @@ - Launch backends in-process or automatically; a test must not depend on a manually started process - Document host prerequisites and the run command in `tests/README.md` + +## Handling untrusted input + +All bytes from the network or a device are attacker-controlled. When parsing: + +- Never index or slice untrusted bytes with manual cursor arithmetic; read + through the `Option`-returning readers (`ByteReader`, `BitReader`) so + bounds-safety is structural + - Do not use panicking accessors (e.g., `bytes::Buf` getters) on + untrusted input +- Bound every buffer that grows across packets or messages with an explicit + cap (e.g., `MAX_PENDING_ACCESS_UNIT_BYTES`), and never let a claimed + length drive an allocation — allocate only for bytes actually received +- Use checked arithmetic on untrusted lengths and bound varint/loop + decoders (LEB128, Exp-Golomb) so they cannot spin +- Malformed input is data, not a crash: return a typed error or engage loss + recovery; keep parser state valid after every error +- Escape or strip untrusted strings before logging them or embedding them + in error messages (strip control characters; log via `{:?}` or + `escape_debug`) +- Never let credentials reach logs, error strings, or request URIs; redact + them in `Debug` implementations +- Insecure opt-outs (e.g., disabled TLS verification) must be explicit + configuration, documented as such, and warned about at runtime +- Prefer well-maintained crates for standard wire grammars (RTSP, SDP, + auth, TLS); keep only domain interpretation in this crate +- Keep (de)serialization in small pure functions with unit tests, so they + stay auditable and fuzzable From a44bd318fdea80ab4032f619c5a22b677c407cc0 Mon Sep 17 00:00:00 2001 From: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:02:13 +0000 Subject: [PATCH 37/38] generated protobuf --- .../proto/capture_pb.d.ts | 91 +++++++++++++++++++ livekit-ffi-node-bindings/proto/capture_pb.js | 21 +++++ 2 files changed, 112 insertions(+) diff --git a/livekit-ffi-node-bindings/proto/capture_pb.d.ts b/livekit-ffi-node-bindings/proto/capture_pb.d.ts index abba9e80d..d08b41a2e 100644 --- a/livekit-ffi-node-bindings/proto/capture_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/capture_pb.d.ts @@ -201,6 +201,91 @@ export declare class GstreamerVideoSourceConfig extends Message | undefined, b: GstreamerVideoSourceConfig | PlainMessage | undefined): boolean; } +/** + * Encoded ingest from an RTSP server over TCP-interleaved RTP. + * + * @generated from message livekit.proto.RtspVideoSourceConfig + */ +export declare class RtspVideoSourceConfig extends Message { + /** + * RTSP URL (rtsp://host[:port]/path, or rtsps:// when the server is built + * with TLS support). URL userinfo is accepted and stripped from requests; + * `username`/`password` take precedence over it. + * + * @generated from field: required string url = 1; + */ + url?: string; + + /** + * Username for RTSP authentication, overriding URL userinfo. + * + * @generated from field: optional string username = 2; + */ + username?: string; + + /** + * Password for RTSP authentication, overriding URL userinfo. + * + * @generated from field: optional string password = 3; + */ + password?: string; + + /** + * Codec required from the stream; the first supported video track offered + * by the SDP is used when omitted. + * + * @generated from field: optional livekit.proto.VideoCodec codec = 4; + */ + codec?: VideoCodec; + + /** + * Encoded frame resolution. When omitted, it is discovered from the SDP + * when declared there, and from the stream's first keyframe otherwise; + * when set, the first keyframe is verified against it. + * + * @generated from field: optional livekit.proto.VideoSourceResolution resolution = 5; + */ + resolution?: VideoSourceResolution; + + /** + * TCP connect and RTSP handshake timeout in milliseconds (default 10000). + * + * @generated from field: optional uint32 connect_timeout_ms = 6; + */ + connectTimeoutMs?: number; + + /** + * Maximum tolerated stream silence in milliseconds (default 30000). + * + * @generated from field: optional uint32 idle_timeout_ms = 7; + */ + idleTimeoutMs?: number; + + /** + * Disables TLS certificate verification for rtsps:// URLs. Most cameras + * present self-signed certificates, which fail verification against the + * system roots. The connection stays encrypted but is not authenticated: + * a network attacker could impersonate the camera. + * + * @generated from field: optional bool accept_invalid_tls_certs = 8; + */ + acceptInvalidTlsCerts?: boolean; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.RtspVideoSourceConfig"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): RtspVideoSourceConfig; + + static fromJson(jsonValue: JsonValue, options?: Partial): RtspVideoSourceConfig; + + static fromJsonString(jsonString: string, options?: Partial): RtspVideoSourceConfig; + + static equals(a: RtspVideoSourceConfig | PlainMessage | undefined, b: RtspVideoSourceConfig | PlainMessage | undefined): boolean; +} + /** * Test pattern rendered on the GPU. * @@ -394,6 +479,12 @@ export declare class NewCaptureSourceRequest extends Message [ + { no: 1, name: "url", kind: "scalar", T: 9 /* ScalarType.STRING */, req: true }, + { no: 2, name: "username", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 3, name: "password", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 4, name: "codec", kind: "enum", T: proto2.getEnumType(VideoCodec), opt: true }, + { no: 5, name: "resolution", kind: "message", T: VideoSourceResolution, opt: true }, + { no: 6, name: "connect_timeout_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true }, + { no: 7, name: "idle_timeout_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true }, + { no: 8, name: "accept_invalid_tls_certs", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + ], +); + /** * Test pattern rendered on the GPU. * @@ -174,6 +193,7 @@ const NewCaptureSourceRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 1, name: "gstreamer", kind: "message", T: GstreamerVideoSourceConfig, oneof: "config" }, { no: 2, name: "pattern", kind: "message", T: PatternVideoSourceConfig, oneof: "config" }, { no: 5, name: "clock", kind: "message", T: ClockVideoSourceConfig, oneof: "config" }, + { no: 6, name: "rtsp", kind: "message", T: RtspVideoSourceConfig, oneof: "config" }, { no: 3, name: "request_async_id", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true }, ], ); @@ -289,6 +309,7 @@ exports.CaptureSourceKind = CaptureSourceKind; exports.CaptureExit = CaptureExit; exports.GstreamerRateControl = GstreamerRateControl; exports.GstreamerVideoSourceConfig = GstreamerVideoSourceConfig; +exports.RtspVideoSourceConfig = RtspVideoSourceConfig; exports.PatternVideoSourceConfig = PatternVideoSourceConfig; exports.ClockVideoSourceConfig = ClockVideoSourceConfig; exports.CaptureSourceInfo = CaptureSourceInfo; From cd1a226096d7cb6e716a783d7a6ba615a1f490d1 Mon Sep 17 00:00:00 2001 From: Jacob Gelman <3182119+ladvoc@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:37:31 -0700 Subject: [PATCH 38/38] Use spawn_blocking directly --- livekit-capture/src/sources/rtsp/mod.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs index 2b82b4d2f..5cb53efe9 100644 --- a/livekit-capture/src/sources/rtsp/mod.rs +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -297,9 +297,12 @@ impl RtspVideoSource { /// /// Requires a running tokio runtime. Use /// [`RtspVideoSource::new_blocking`] outside of async contexts. - #[cfg(feature = "tokio")] pub async fn new(config: RtspVideoSourceConfig) -> Result { - crate::utils::run_blocking(move || Self::new_blocking(config)).await + match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(SourceError::new(err)), + } } /// Connects to the RTSP server and starts playback.