diff --git a/.changeset/capture-source-rtsp.md b/.changeset/capture-source-rtsp.md new file mode 100644 index 000000000..6b2e9f849 --- /dev/null +++ b/.changeset/capture-source-rtsp.md @@ -0,0 +1,6 @@ +--- +livekit-capture: minor +livekit-ffi: minor +--- + +Add a capture source that ingests encoded video from an RTSP server. diff --git a/Cargo.lock b/Cargo.lock index 73b434a8a..9b343c144 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,6 +735,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "bstr" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f" +dependencies = [ + "memchr", +] + [[package]] name = "built" version = "0.8.1" @@ -1162,6 +1171,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "cookie-factory" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9885fa71e26b8ab7855e2ec7cae6e9b380edff76cd052e07c683a0319d51b3a2" +dependencies = [ + "futures", +] + [[package]] name = "core-foundation" version = "0.7.0" @@ -2056,6 +2074,12 @@ dependencies = [ "rand 0.9.5", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + [[package]] name = "fastrand" version = "2.5.0" @@ -2437,6 +2461,23 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "gio" +version = "0.22.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b3e1f669909c326b9413bde5a742097b8c90a7d78f45326db13668984769ded" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-util", + "gio-sys 0.22.8", + "glib 0.22.8", + "libc", + "pin-project-lite", + "smallvec", +] + [[package]] name = "gio-sys" version = "0.21.5" @@ -2751,6 +2792,115 @@ dependencies = [ "system-deps", ] +[[package]] +name = "gstreamer-net" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abcad04d471a4f2c859ef1287f22581b07064e357cc89dfa415ffc99b2a3193d" +dependencies = [ + "gio", + "glib 0.22.8", + "gstreamer", + "gstreamer-net-sys", +] + +[[package]] +name = "gstreamer-net-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcefb342a98ffca0b106bc9fea13d5df15879e1613b4dc652a6ec1960c32584c" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-rtsp" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deeb8cb21db41f90402f4de49403f44492b737e872a06665b93d90987a1bacaa" +dependencies = [ + "glib 0.22.8", + "gstreamer", + "gstreamer-rtsp-sys", + "gstreamer-sdp", + "libc", +] + +[[package]] +name = "gstreamer-rtsp-server" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "379699a4451e4f73b1f19a11d6640c631d308bca130628161802d34e429a6c03" +dependencies = [ + "gio", + "glib 0.22.8", + "gstreamer", + "gstreamer-net", + "gstreamer-rtsp", + "gstreamer-rtsp-server-sys", + "gstreamer-sdp", + "libc", +] + +[[package]] +name = "gstreamer-rtsp-server-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca124fc3e4803fe31d618e29e53a66f46a442178c37303fccaea0f999366221c" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-net-sys", + "gstreamer-rtsp-sys", + "gstreamer-sdp-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-rtsp-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36b59f39a4577720af3ca5a4ca4402b8ee5ff96b391c8895d1b28ba481a3ad" +dependencies = [ + "gio-sys 0.22.8", + "glib-sys 0.22.8", + "gobject-sys 0.22.6", + "gstreamer-sdp-sys", + "gstreamer-sys", + "libc", + "system-deps", +] + +[[package]] +name = "gstreamer-sdp" +version = "0.25.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a7859290ba340f1bd45d4bc0ada958700e9d3b1a522b456fd7ee88da468cd13" +dependencies = [ + "glib 0.22.8", + "gstreamer", + "gstreamer-sdp-sys", +] + +[[package]] +name = "gstreamer-sdp-sys" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20f0eb41ecfbacbf6a29d9457e6de0f59e2638f47fbdb6a6c1bcfb720c2b9ee" +dependencies = [ + "glib-sys 0.22.8", + "gstreamer-sys", + "libc", + "system-deps", +] + [[package]] name = "gstreamer-sys" version = "0.25.2" @@ -2891,6 +3041,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + [[package]] name = "hexf-parse" version = "0.2.1" @@ -2942,6 +3098,21 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "base64 0.22.1", + "digest", + "hex", + "md-5", + "memchr", + "rand 0.8.7", + "sha2", +] + [[package]] name = "http-body" version = "0.4.6" @@ -3828,17 +3999,30 @@ dependencies = [ name = "livekit-capture" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "chrono", + "gio", "gstreamer", "gstreamer-app", + "gstreamer-rtsp", + "gstreamer-rtsp-server", + "http-auth", "livekit", "log", "pollster", + "rcgen", + "rtsp-types", + "rustls", + "rustls-native-certs", + "rustls-pki-types", "schemars", + "sdp-types", "serde", + "test-log", "thiserror 2.0.19", "tokio", + "url", "wgpu", "yuv-sys", ] @@ -4201,6 +4385,16 @@ dependencies = [ "rayon", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if 1.0.4", + "digest", +] + [[package]] name = "memchr" version = "2.8.3" @@ -5403,6 +5597,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pem" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" +dependencies = [ + "base64 0.22.1", + "serde_core", +] + [[package]] name = "peniko" version = "0.6.1" @@ -6147,6 +6351,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "rcgen" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" +dependencies = [ + "pem", + "ring", + "rustls-pki-types", + "time", + "yasna", +] + [[package]] name = "read-fonts" version = "0.39.2" @@ -6368,6 +6585,18 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153" +[[package]] +name = "rtsp-types" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f47bf238c3da7994ef66a71724f385cb1ee25dcf04a156402d2727ad84dc1433" +dependencies = [ + "cookie-factory", + "nom 7.1.3", + "tinyvec", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.28" @@ -6630,6 +6859,18 @@ dependencies = [ "tiny-skia", ] +[[package]] +name = "sdp-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46d46be065d0565cf5c6703b6ba9f92b0de85b20a4c83f87e110bc0a1040b1bc" +dependencies = [ + "bstr", + "fallible-iterator", + "hex", + "thiserror 2.0.19", +] + [[package]] name = "security-framework" version = "3.7.0" @@ -9382,6 +9623,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" +[[package]] +name = "yasna" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +dependencies = [ + "time", +] + [[package]] name = "yoke" version = "0.8.3" diff --git a/livekit-capture/AGENTS.md b/livekit-capture/AGENTS.md index 19fa12189..af0927821 100644 --- a/livekit-capture/AGENTS.md +++ b/livekit-capture/AGENTS.md @@ -20,3 +20,53 @@ - NEVER add a source that is not consumable uniformly through one of these traits - Doing so would break API contract and break integration for consumers - Keep API surface minimal and hide implementation details + +## Adding integration tests + +- Integration tests verify a source against a real backend (e.g., a real + RTSP server, a real capture device) and never run by default +- Gate each source's tests behind an internal feature named + `__test-source-` (e.g., `__test-source-rtsp`) that enables + `source-` plus any test-only dependencies + - Test-only dependencies are optional entries in `[dependencies]` enabled + only by the test feature (Cargo has no optional dev-dependencies) +- Tests live in `tests/source__test.rs`, with + `#![cfg(feature = "__test-source-")]` at the top so the target + compiles to empty without the feature +- Helpers live in `tests/common/`: source-agnostic ones (e.g., pull loops) + in `tests/common/mod.rs`, and source-specific ones (test servers, + pipelines) in a `tests/common/.rs` submodule declared with + `#[cfg(feature = "__test-source-")]` +- Test the source through its public API (construct, `next_access_unit` or + `next_frame`); do not publish to an RTC source or a LiveKit server +- Launch backends in-process or automatically; a test must not depend on a + manually started process +- Document host prerequisites and the run command in `tests/README.md` + +## Handling untrusted input + +All bytes from the network or a device are attacker-controlled. When parsing: + +- Never index or slice untrusted bytes with manual cursor arithmetic; read + through the `Option`-returning readers (`ByteReader`, `BitReader`) so + bounds-safety is structural + - Do not use panicking accessors (e.g., `bytes::Buf` getters) on + untrusted input +- Bound every buffer that grows across packets or messages with an explicit + cap (e.g., `MAX_PENDING_ACCESS_UNIT_BYTES`), and never let a claimed + length drive an allocation — allocate only for bytes actually received +- Use checked arithmetic on untrusted lengths and bound varint/loop + decoders (LEB128, Exp-Golomb) so they cannot spin +- Malformed input is data, not a crash: return a typed error or engage loss + recovery; keep parser state valid after every error +- Escape or strip untrusted strings before logging them or embedding them + in error messages (strip control characters; log via `{:?}` or + `escape_debug`) +- Never let credentials reach logs, error strings, or request URIs; redact + them in `Debug` implementations +- Insecure opt-outs (e.g., disabled TLS verification) must be explicit + configuration, documented as such, and warned about at runtime +- Prefer well-maintained crates for standard wire grammars (RTSP, SDP, + auth, TLS); keep only domain interpretation in this crate +- Keep (de)serialization in small pure functions with unit tests, so they + stay auditable and fuzzable diff --git a/livekit-capture/Cargo.toml b/livekit-capture/Cargo.toml index 711665cb7..8c2b1c3ab 100644 --- a/livekit-capture/Cargo.toml +++ b/livekit-capture/Cargo.toml @@ -8,10 +8,23 @@ edition.workspace = true repository.workspace = true [dependencies] +base64 = { version = "0.22", optional = true } bytes = { workspace = true } chrono = { version = "0.4", default-features = false, features = ["clock"], optional = true } gstreamer = { version = "0.25.2", optional = true } gstreamer-app = { version = "0.25.2", optional = true } +# Test-only (`__test-source-rtsp`); in `[dependencies]` because Cargo has no +# optional dev-dependencies. +gio = { version = "0.22", optional = true } +gstreamer-rtsp = { version = "0.25", optional = true } +gstreamer-rtsp-server = { version = "0.25", optional = true } +http-auth = { version = "0.1", optional = true } +rtsp-types = { version = "0.1", optional = true } +rustls = { version = "0.23", default-features = false, features = ["ring", "logging", "std", "tls12"], optional = true } +rustls-native-certs = { version = "0.8", optional = true } +rustls-pki-types = { version = "1", optional = true } +sdp-types = { version = "0.2", optional = true } +url = { version = "2", optional = true } livekit = { workspace = true } log = { workspace = true } pollster = { version = "0.4", optional = true } @@ -23,6 +36,9 @@ wgpu = { workspace = true, optional = true } yuv-sys = { workspace = true, features = ["jpeg"], optional = true } [dev-dependencies] +log = { workspace = true } +rcgen = "0.13" +test-log = "0.2" tokio = { workspace = true, features = ["rt", "time", "macros"] } [features] @@ -36,3 +52,20 @@ source-pattern = ["dep:pollster", "dep:wgpu", "dep:yuv-sys"] # Encoded sources source-gstreamer = ["dep:gstreamer", "dep:gstreamer-app"] +source-rtsp = ["dep:base64", "dep:http-auth", "dep:rtsp-types", "dep:sdp-types", "dep:url"] +# Adds `rtsps://` (RTSP over TLS, 1.2+) support to the RTSP source. +# Certificates are verified against the system roots by default. +source-rtsp-tls = ["source-rtsp", "dep:rustls", "dep:rustls-native-certs", "dep:rustls-pki-types"] + +# Testing +# In-process RTSP server for `tests/source_rtsp_test.rs`. Requires the +# system GStreamer RTSP server library. Test-only; not part of the public +# API. +__test-source-rtsp = [ + "source-rtsp", + "source-rtsp-tls", + "dep:gio", + "dep:gstreamer", + "dep:gstreamer-rtsp", + "dep:gstreamer-rtsp-server", +] diff --git a/livekit-capture/README.md b/livekit-capture/README.md index aa3541208..d60db3203 100644 --- a/livekit-capture/README.md +++ b/livekit-capture/README.md @@ -57,9 +57,14 @@ named `source-`. Each module documents its source. | Feature | Source | Kind | | ------------------ | ---------------------- | ------- | | `source-gstreamer` | `GStreamerVideoSource` | encoded | +| `source-rtsp` | `RtspVideoSource` | encoded | | `source-pattern` | `PatternVideoSource` | pixel | | `source-clock` | `ClockVideoSource` | pixel | +`source-rtsp-tls` extends `RtspVideoSource` with `rtsps://` support (RTSP +over TLS 1.2+). Certificates are verified against the system roots by +default; cameras with self-signed certificates can opt out per source. + ## Custom sources Implement `pixel::PixelVideoSource` for raw frames or diff --git a/livekit-capture/src/encoded/h26x.rs b/livekit-capture/src/encoded/h26x.rs index 2790a0db4..af5e995f8 100644 --- a/livekit-capture/src/encoded/h26x.rs +++ b/livekit-capture/src/encoded/h26x.rs @@ -44,7 +44,7 @@ pub enum H26xParseError { const ANNEX_B_START_CODE: [u8; 4] = [0, 0, 0, 1]; /// Upper bound on bytes buffered while waiting for an access-unit boundary. -const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; +pub(crate) const MAX_PENDING_ACCESS_UNIT_BYTES: usize = 32 * 1024 * 1024; /// Byte-stream access-unit parser shared by the encoded ingest sources. /// @@ -454,9 +454,9 @@ fn is_keyframe_nalus( nal_units: &[&[u8]], ) -> Result { match codec { - EncodedVideoCodec::H264 => { - nal_units.iter().try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == 5)) - } + EncodedVideoCodec::H264 => nal_units + .iter() + .try_fold(false, |is_key, nal| Ok(is_key || h264_nal_type(nal)? == H264_NAL_IDR)), EncodedVideoCodec::H265 => { let mut has_vps = false; let mut has_sps = false; @@ -465,10 +465,10 @@ fn is_keyframe_nalus( for nal in nal_units { match h265_nal_type(nal)? { - 32 => has_vps = true, - 33 => has_sps = true, - 34 => has_pps = true, - 19 | 20 => has_idr = true, + H265_NAL_VPS => has_vps = true, + H265_NAL_SPS => has_sps = true, + H265_NAL_PPS => has_pps = true, + H265_NAL_IDR_W_RADL | H265_NAL_IDR_N_LP => has_idr = true, _ => {} } } @@ -481,12 +481,26 @@ fn is_keyframe_nalus( } } -fn h264_nal_type(nal: &[u8]) -> Result { +/// H.264 NAL unit types (ITU-T H.264 table 7-1). +pub(crate) const H264_NAL_IDR: u8 = 5; +pub(crate) const H264_NAL_SPS: u8 = 7; +pub(crate) const H264_NAL_PPS: u8 = 8; + +/// H.265 NAL unit types (ITU-T H.265 table 7-1). +pub(crate) const H265_NAL_IDR_W_RADL: u8 = 19; +pub(crate) const H265_NAL_IDR_N_LP: u8 = 20; +pub(crate) const H265_NAL_VPS: u8 = 32; +pub(crate) const H265_NAL_SPS: u8 = 33; +pub(crate) const H265_NAL_PPS: u8 = 34; + +/// Extracts the type of an H.264 NAL unit from its header. +pub(crate) fn h264_nal_type(nal: &[u8]) -> Result { let header = nal.first().ok_or(H26xParseError::EmptyPayload)?; Ok(header & 0x1f) } -fn h265_nal_type(nal: &[u8]) -> Result { +/// Extracts the type of an H.265 NAL unit from its header. +pub(crate) fn h265_nal_type(nal: &[u8]) -> Result { if nal.is_empty() { return Err(H26xParseError::EmptyPayload); } @@ -635,7 +649,9 @@ fn find_start_code(bytes: &[u8]) -> Option<(usize, usize)> { None } -fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { +/// Splits a sequence of length-prefixed NAL units, as used by AVC-format +/// access units and, with a 2-byte length, RTP aggregation packets. +pub(crate) fn avc_nalus(payload: &[u8], nal_length_size: u8) -> Result, H26xParseError> { let ranges = avc_nal_ranges(payload, nal_length_size, true)?; if ranges.is_empty() { return Err(H26xParseError::EmptyPayload); diff --git a/livekit-capture/src/sources/mod.rs b/livekit-capture/src/sources/mod.rs index 732fb8bdd..377a0ff73 100644 --- a/livekit-capture/src/sources/mod.rs +++ b/livekit-capture/src/sources/mod.rs @@ -23,3 +23,6 @@ pub mod gstreamer; #[cfg(feature = "source-pattern")] pub mod pattern; + +#[cfg(feature = "source-rtsp")] +pub mod rtsp; diff --git a/livekit-capture/src/sources/rtsp/auth.rs b/livekit-capture/src/sources/rtsp/auth.rs new file mode 100644 index 000000000..5e9e6556c --- /dev/null +++ b/livekit-capture/src/sources/rtsp/auth.rs @@ -0,0 +1,199 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP authentication, backed by the `http-auth` crate: Basic and Digest +//! (RFC 7616, including SHA-256 and session variants). + +use std::fmt; + +use super::{client::RtspResponse, RtspVideoSourceError}; + +/// Username and password for RTSP authentication. +#[derive(Clone, PartialEq, Eq)] +pub(super) struct RtspCredentials { + pub(super) username: String, + pub(super) password: String, +} + +// Manual so a `{:?}` of the source or its context never prints the password. +impl fmt::Debug for RtspCredentials { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspCredentials") + .field("username", &self.username) + .field("password", &"") + .finish() + } +} + +/// Tracks the server's authentication challenge across requests. +pub(super) struct RtspAuthContext { + credentials: Option, + client: Option, +} + +impl RtspAuthContext { + /// Creates an authentication context; without credentials, a challenge + /// fails with [`RtspVideoSourceError::MissingCredentials`]. + pub(super) fn new(credentials: Option) -> Self { + Self { credentials, client: None } + } + + /// Builds the `Authorization` header value for a request, once the server + /// has issued a challenge. + pub(super) fn header( + &mut self, + method: &str, + uri: &str, + ) -> Result, RtspVideoSourceError> { + let Some(client) = self.client.as_mut() else { + return Ok(None); + }; + let credentials = + self.credentials.as_ref().ok_or(RtspVideoSourceError::MissingCredentials)?; + let authorization = client + .respond(&http_auth::PasswordParams { + username: &credentials.username, + password: &credentials.password, + uri, + method, + // RTSP requests carry no body relevant to `auth-int`. + body: Some(&[]), + }) + .map_err(|err| RtspVideoSourceError::Auth(super::sanitized(err)))?; + Ok(Some(authorization)) + } + + /// Ingests the challenges of a 401 response so the retry can + /// authenticate. Digest is preferred over Basic when both are offered. + pub(super) fn update_from_unauthorized( + &mut self, + response: &RtspResponse, + ) -> Result<(), RtspVideoSourceError> { + if self.credentials.is_none() { + return Err(RtspVideoSourceError::MissingCredentials); + } + let mut builder = http_auth::PasswordClient::builder(); + for challenges in response.headers("www-authenticate") { + builder = builder.challenges(challenges); + } + // Build errors can quote the server's challenge bytes. + self.client = + Some(builder.build().map_err(|err| RtspVideoSourceError::Auth(super::sanitized(err)))?); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn context_with_credentials() -> RtspAuthContext { + RtspAuthContext::new(Some(RtspCredentials { + username: "user".to_owned(), + password: "pass".to_owned(), + })) + } + + fn unauthorized(challenges: &[&str]) -> RtspResponse { + let mut bytes = b"RTSP/1.0 401 Unauthorized\r\nCSeq: 1\r\n".to_vec(); + for challenge in challenges { + bytes.extend_from_slice(format!("WWW-Authenticate: {challenge}\r\n").as_bytes()); + } + bytes.extend_from_slice(b"\r\n"); + RtspResponse::parse_for_tests(&bytes) + } + + #[test] + fn answers_basic_challenge() { + let mut context = context_with_credentials(); + context.update_from_unauthorized(&unauthorized(&["Basic realm=\"camera\""])).unwrap(); + + let header = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + assert_eq!(header, "Basic dXNlcjpwYXNz"); + } + + #[test] + fn prefers_digest_over_basic() { + let mut context = context_with_credentials(); + context + .update_from_unauthorized(&unauthorized(&[ + "Basic realm=\"camera\"", + "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"", + ])) + .unwrap(); + + let header = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + assert!(header.starts_with("Digest "), "unexpected header: {header}"); + assert!(header.contains("username=\"user\"")); + assert!(header.contains("nc=00000001")); + } + + #[test] + fn digest_nonce_count_increments_across_requests() { + let mut context = context_with_credentials(); + context + .update_from_unauthorized(&unauthorized(&[ + "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"", + ])) + .unwrap(); + + let _ = context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().unwrap(); + let second = context.header("SETUP", "rtsp://camera.example/live").unwrap().unwrap(); + assert!(second.contains("nc=00000002"), "unexpected header: {second}"); + } + + #[test] + fn rejects_unsupported_scheme() { + let mut context = context_with_credentials(); + let err = + context.update_from_unauthorized(&unauthorized(&["Bearer token=\"abc\""])).unwrap_err(); + assert!(matches!(err, RtspVideoSourceError::Auth(_)), "unexpected error: {err:?}"); + } + + #[test] + fn escapes_control_characters_in_auth_errors() { + let mut context = context_with_credentials(); + let err = context + .update_from_unauthorized(&unauthorized(&["Bearer \u{7f}\u{1b}[31mfake"])) + .unwrap_err(); + let message = err.to_string(); + assert!( + !message.chars().any(char::is_control), + "control characters in message: {message:?}" + ); + } + + #[test] + fn requires_credentials_for_challenges() { + let mut context = RtspAuthContext::new(None); + let err = + context.update_from_unauthorized(&unauthorized(&["Basic realm=\"c\""])).unwrap_err(); + assert!(matches!(err, RtspVideoSourceError::MissingCredentials)); + } + + #[test] + fn no_header_before_challenge() { + let mut context = context_with_credentials(); + assert!(context.header("DESCRIBE", "rtsp://camera.example/live").unwrap().is_none()); + } + + #[test] + fn debug_redacts_password() { + let credentials = + RtspCredentials { username: "admin".to_owned(), password: "secret".to_owned() }; + let debug = format!("{credentials:?}"); + assert!(debug.contains("admin")); + assert!(!debug.contains("secret")); + } +} diff --git a/livekit-capture/src/sources/rtsp/bits.rs b/livekit-capture/src/sources/rtsp/bits.rs new file mode 100644 index 000000000..04f24efd6 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/bits.rs @@ -0,0 +1,237 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Bitstream readers shared by the RTP depacketizers and the codec header +//! parsers. + +/// MSB-first bit reader over a byte slice. Every read returns `None` past +/// the end of the input. +#[derive(Debug, Clone)] +pub(super) struct BitReader<'a> { + bytes: &'a [u8], + bit_offset: usize, +} + +impl<'a> BitReader<'a> { + /// Creates a reader positioned at the first bit. + pub(super) fn new(bytes: &'a [u8]) -> Self { + Self { bytes, bit_offset: 0 } + } + + /// Reads one bit. + pub(super) fn read_bit(&mut self) -> Option { + let byte = *self.bytes.get(self.bit_offset / 8)?; + let bit = (byte >> (7 - self.bit_offset % 8)) & 0x01; + self.bit_offset += 1; + Some(bit) + } + + /// Reads one bit as a flag. + pub(super) fn read_flag(&mut self) -> Option { + self.read_bit().map(|bit| bit != 0) + } + + /// Reads up to 32 bits MSB-first. + pub(super) fn read_bits(&mut self, bits: u32) -> Option { + debug_assert!(bits <= 32); + let mut value = 0u32; + for _ in 0..bits { + value = (value << 1) | u32::from(self.read_bit()?); + } + Some(value) + } + + /// Skips over `bits` bits. + pub(super) fn skip_bits(&mut self, bits: usize) -> Option<()> { + let next = self.bit_offset.checked_add(bits)?; + if next > self.bytes.len() * 8 { + return None; + } + self.bit_offset = next; + Some(()) + } + + /// Reads an unsigned Exp-Golomb code (`ue(v)` in H.264/H.265). + pub(super) fn read_ue(&mut self) -> Option { + let mut leading_zeros = 0u32; + while self.read_bit()? == 0 { + leading_zeros += 1; + if leading_zeros > 31 { + return None; + } + } + let suffix = self.read_bits(leading_zeros)?; + (1u32 << leading_zeros).checked_sub(1)?.checked_add(suffix) + } + + /// Reads a signed Exp-Golomb code (`se(v)` in H.264/H.265) and discards + /// the value. + pub(super) fn skip_se(&mut self) -> Option<()> { + self.read_ue().map(|_| ()) + } +} + +/// Byte-level reader over untrusted input. Every read returns `None` past +/// the end of the input, so callers cannot index out of bounds. +#[derive(Debug, Clone)] +pub(super) struct ByteReader<'a> { + bytes: &'a [u8], +} + +impl<'a> ByteReader<'a> { + /// Creates a reader positioned at the first byte. + pub(super) fn new(bytes: &'a [u8]) -> Self { + Self { bytes } + } + + /// Returns `true` when all input was consumed. + pub(super) fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// Reads one byte. + pub(super) fn get_u8(&mut self) -> Option { + let (&byte, rest) = self.bytes.split_first()?; + self.bytes = rest; + Some(byte) + } + + /// Reads a big-endian `u16`. + pub(super) fn get_u16_be(&mut self) -> Option { + let bytes = self.take(2)?; + Some(u16::from_be_bytes([bytes[0], bytes[1]])) + } + + /// Reads a big-endian `u32`. + pub(super) fn get_u32_be(&mut self) -> Option { + let bytes = self.take(4)?; + Some(u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]])) + } + + /// Reads an AV1/LEB128-encoded length. + pub(super) fn get_leb128(&mut self) -> Option { + let mut value = 0usize; + let mut shift = 0usize; + loop { + let byte = self.get_u8()?; + value |= usize::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Some(value); + } + shift += 7; + if shift >= usize::BITS as usize { + return None; + } + } + } + + /// Reads the next `len` bytes. + pub(super) fn take(&mut self, len: usize) -> Option<&'a [u8]> { + if len > self.bytes.len() { + return None; + } + let (taken, rest) = self.bytes.split_at(len); + self.bytes = rest; + Some(taken) + } + + /// Skips over the next `len` bytes. + pub(super) fn skip(&mut self, len: usize) -> Option<()> { + self.take(len).map(|_| ()) + } + + /// Consumes and returns the unread remainder. + pub(super) fn take_rest(&mut self) -> &'a [u8] { + std::mem::take(&mut self.bytes) + } +} + +/// Appends `value` to `out` as LEB128. +pub(super) fn write_leb128(mut value: usize, out: &mut Vec) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + out.push(byte); + if value == 0 { + break; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_bits_msb_first() { + let mut reader = BitReader::new(&[0b1011_0001, 0b1000_0000]); + assert_eq!(reader.read_bit(), Some(1)); + assert_eq!(reader.read_bits(3), Some(0b011)); + assert_eq!(reader.read_bits(5), Some(0b0001_1)); + assert!(reader.skip_bits(7).is_some()); + assert_eq!(reader.read_bit(), None); + } + + #[test] + fn reads_exp_golomb_codes() { + // ue(v) codes: 1 -> 0, 010 -> 1, 011 -> 2, 00100 -> 3. + let mut reader = BitReader::new(&[0b1010_0110, 0b0100_0000]); + assert_eq!(reader.read_ue(), Some(0)); + assert_eq!(reader.read_ue(), Some(1)); + assert_eq!(reader.read_ue(), Some(2)); + assert_eq!(reader.read_ue(), Some(3)); + } + + #[test] + fn exp_golomb_past_end_is_none() { + let mut reader = BitReader::new(&[0b0000_0000]); + assert_eq!(reader.read_ue(), None); + } + + #[test] + fn byte_reader_reads_and_bounds() { + let mut reader = ByteReader::new(&[1, 0, 2, 0, 0, 0, 3, 4, 5, 6]); + assert_eq!(reader.get_u8(), Some(1)); + assert_eq!(reader.get_u16_be(), Some(2)); + assert_eq!(reader.get_u32_be(), Some(3)); + assert_eq!(reader.take(2), Some(&[4, 5][..])); + assert!(reader.skip(2).is_none()); + assert_eq!(reader.take_rest(), &[6]); + assert!(reader.is_empty()); + assert_eq!(reader.get_u8(), None); + } + + #[test] + fn byte_reader_reads_leb128() { + let mut reader = ByteReader::new(&[0xac, 0x02, 0x00]); + assert_eq!(reader.get_leb128(), Some(300)); + assert_eq!(reader.get_leb128(), Some(0)); + assert_eq!(reader.get_leb128(), None); + assert_eq!(ByteReader::new(&[0x80]).get_leb128(), None); + } + + #[test] + fn leb128_round_trips() { + for value in [0usize, 1, 127, 128, 300, 16_383, 16_384, usize::from(u16::MAX)] { + let mut encoded = Vec::new(); + write_leb128(value, &mut encoded); + let mut reader = ByteReader::new(&encoded); + assert_eq!(reader.get_leb128(), Some(value)); + assert!(reader.is_empty()); + } + } +} diff --git a/livekit-capture/src/sources/rtsp/client.rs b/livekit-capture/src/sources/rtsp/client.rs new file mode 100644 index 000000000..9eddbc4c1 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/client.rs @@ -0,0 +1,899 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP/1.0 client over TCP: request writing, response reading, and the +//! interleaved (`$`-framed) stream demuxer. + +use std::{ + fmt, + io::{self, Read, Write}, + net::{TcpStream, ToSocketAddrs}, + str, + time::{Duration, Instant}, +}; + +use bytes::{Buf, Bytes, BytesMut}; +use rtsp_types::{headers, HeaderName, Message, Method, ParseError, Url, Version}; + +#[cfg(feature = "source-rtsp-tls")] +use super::sanitized; +use super::{auth::RtspAuthContext, auth::RtspCredentials, RtspPhase, RtspVideoSourceError}; + +/// Socket read timeout: the poll granularity for the stop token, keepalives, +/// and deadlines, kept near one frame interval per the pump contract. +const READ_POLL: Duration = Duration::from_millis(100); + +/// Socket write timeout; requests are small, so a stalled write means the +/// connection is gone. +const WRITE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Upper bound on one buffered-but-incomplete RTSP message. Interleaved +/// frames are at most 4 + 65535 bytes, and responses are far smaller, so an +/// incomplete message larger than this means the framing is corrupt. +const MAX_PENDING_MESSAGE_BYTES: usize = 128 * 1024; + +/// Bytes requested from the socket per read. +const READ_CHUNK_BYTES: usize = 8 * 1024; + +/// A parsed `rtsp://` or `rtsps://` URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct RtspUrl { + /// Request URI with any userinfo stripped, so credentials never appear + /// on the wire outside the `Authorization` header. + pub(super) request_uri: String, + /// Credentials from the URL userinfo, percent-decoded. + pub(super) credentials: Option, + /// Whether the URL requires TLS (`rtsps://`). + pub(super) tls: bool, + connect_host: String, + port: u16, +} + +impl RtspUrl { + /// Parses an `rtsp(s)://[user:password@]host[:port][/path]` URL. + pub(super) fn parse(url: &str) -> Result { + let parsed = + Url::parse(url).map_err(|_| RtspVideoSourceError::InvalidUrl("malformed URL"))?; + let tls = match parsed.scheme() { + "rtsp" => false, + "rtsps" => true, + _ => { + return Err(RtspVideoSourceError::InvalidUrl( + "expected rtsp:// or rtsps:// scheme", + )) + } + }; + + // `Host::Ipv6` renders unbracketed, as `ToSocketAddrs` expects. + let connect_host = match parsed.host() { + Some(url::Host::Domain(domain)) if !domain.is_empty() => domain.to_owned(), + Some(url::Host::Ipv4(address)) => address.to_string(), + Some(url::Host::Ipv6(address)) => address.to_string(), + _ => return Err(RtspVideoSourceError::InvalidUrl("missing host")), + }; + let port = parsed.port().unwrap_or(if tls { 322 } else { 554 }); + + let credentials = if parsed.username().is_empty() && parsed.password().is_none() { + None + } else { + let username = percent_decode(parsed.username()); + if username.is_empty() { + return Err(RtspVideoSourceError::InvalidUrl("missing username")); + } + let password = parsed.password().map(percent_decode).unwrap_or_default(); + Some(RtspCredentials { username, password }) + }; + + // Strip userinfo so credentials never appear on the wire outside + // the `Authorization` header. + let mut request_uri = parsed; + let _ = request_uri.set_username(""); + let _ = request_uri.set_password(None); + + Ok(Self { request_uri: request_uri.to_string(), credentials, tls, connect_host, port }) + } +} + +/// Decodes RFC 3986 percent-escapes; malformed escapes pass through as-is. +fn percent_decode(value: &str) -> String { + let bytes = value.as_bytes(); + let mut decoded = Vec::with_capacity(bytes.len()); + let mut cursor = 0; + while cursor < bytes.len() { + let escape = (bytes[cursor] == b'%') + .then(|| bytes.get(cursor + 1..cursor + 3)) + .flatten() + .and_then(|digits| str::from_utf8(digits).ok()) + .and_then(|digits| u8::from_str_radix(digits, 16).ok()); + if let Some(byte) = escape { + decoded.push(byte); + cursor += 3; + } else { + decoded.push(bytes[cursor]); + cursor += 1; + } + } + String::from_utf8_lossy(&decoded).into_owned() +} + +/// A parsed RTSP response. +#[derive(Debug, Clone)] +pub(super) struct RtspResponse { + pub(super) status_code: u16, + pub(super) reason: String, + headers: Vec<(String, String)>, + pub(super) body: Vec, +} + +impl RtspResponse { + /// Converts a parsed `rtsp-types` response into the client's view. + fn from_message(response: rtsp_types::Response>) -> Self { + Self { + status_code: response.status().into(), + // The reason phrase flows into error strings and logs; strip + // control characters so a server cannot forge log lines or + // emit terminal escapes. + reason: response.reason_phrase().chars().filter(|c| !c.is_control()).collect(), + headers: response + .headers() + .map(|(name, value)| (name.as_str().to_owned(), value.as_str().to_owned())) + .collect(), + body: response.into_body(), + } + } + + pub(super) fn is_success(&self) -> bool { + (200..300).contains(&self.status_code) + } + + /// Returns the first header with the given name, case-insensitively. + pub(super) fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } + + /// Returns every header with the given name, case-insensitively. + pub(super) fn headers<'a>(&'a self, name: &'a str) -> impl Iterator + 'a { + self.headers + .iter() + .filter(move |(header_name, _)| header_name.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) + } +} + +/// One unit read from the interleaved stream. +#[derive(Debug)] +pub(super) enum InterleavedPoll { + /// An interleaved binary frame. + Frame { + /// Interleaved channel the frame arrived on. + channel: u8, + /// Frame payload, without the 4-byte interleaved header. + payload: Bytes, + }, + /// An in-stream RTSP response, such as a keepalive reply. + Response(RtspResponse), + /// A read timed out; framing state is preserved for the next poll. + TimedOut, + /// The stream ended cleanly at a unit boundary. + EndOfStream, +} + +/// Result of one attempt to read more stream bytes. +enum StreamFill { + Filled, + Eof, + TimedOut, +} + +/// The connection's byte stream: plain TCP, or TLS over TCP for `rtsps://`. +enum Transport { + Plain(TcpStream), + #[cfg(feature = "source-rtsp-tls")] + Tls(Box>), +} + +impl Transport { + /// The underlying TCP socket, whose timeouts bound every read and write. + fn socket(&self) -> &TcpStream { + match self { + Self::Plain(stream) => stream, + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.get_ref(), + } + } +} + +impl Read for Transport { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Plain(stream) => stream.read(buf), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.read(buf), + } + } +} + +impl Write for Transport { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + Self::Plain(stream) => stream.write(buf), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.write(buf), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self { + Self::Plain(stream) => stream.flush(), + #[cfg(feature = "source-rtsp-tls")] + Self::Tls(stream) => stream.flush(), + } + } +} + +/// RTSP connection: owns the transport stream, the read buffer, the request +/// sequence number, and the authentication context. +pub(super) struct RtspClient { + stream: Transport, + buf: BytesMut, + scratch: Vec, + cseq: u32, + auth: RtspAuthContext, + last_read_at: Instant, + logged_server_request: bool, +} + +// Manual so the read buffer's contents are not dumped; the authentication +// context redacts its own credentials. +impl fmt::Debug for RtspClient { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspClient") + .field("cseq", &self.cseq) + .finish_non_exhaustive() + } +} + +impl RtspClient { + /// Connects to the URL's host, bounded by `deadline`, establishes TLS + /// for `rtsps://` URLs, and prepares the socket for polled reads. + pub(super) fn connect( + url: &RtspUrl, + credentials: Option, + accept_invalid_tls_certs: bool, + deadline: Instant, + ) -> Result { + #[cfg(not(feature = "source-rtsp-tls"))] + let _ = accept_invalid_tls_certs; + + let addrs = (url.connect_host.as_str(), url.port).to_socket_addrs()?; + let mut last_error = None; + let mut stream = None; + for addr in addrs { + let Some(remaining) = deadline.checked_duration_since(Instant::now()).filter(|d| !d.is_zero()) else { + break; + }; + match TcpStream::connect_timeout(&addr, remaining) { + Ok(connected) => { + stream = Some(connected); + break; + } + Err(err) => last_error = Some(err), + } + } + let Some(stream) = stream else { + return Err(match last_error { + Some(err) => RtspVideoSourceError::Io(err), + None => RtspVideoSourceError::Timeout { phase: RtspPhase::Connect }, + }); + }; + + if let Err(err) = stream.set_nodelay(true) { + log::debug!("failed to disable Nagle's algorithm on the RTSP stream: {err}"); + } + stream.set_read_timeout(Some(READ_POLL))?; + stream.set_write_timeout(Some(WRITE_TIMEOUT))?; + + let stream = match url.tls { + false => Transport::Plain(stream), + #[cfg(feature = "source-rtsp-tls")] + true => Transport::Tls(Box::new(tls::establish( + stream, + &url.connect_host, + accept_invalid_tls_certs, + deadline, + )?)), + #[cfg(not(feature = "source-rtsp-tls"))] + true => return Err(RtspVideoSourceError::TlsNotSupported), + }; + + Ok(Self { + stream, + buf: BytesMut::with_capacity(READ_CHUNK_BYTES), + scratch: vec![0; READ_CHUNK_BYTES], + cseq: 1, + auth: RtspAuthContext::new(credentials), + last_read_at: Instant::now(), + logged_server_request: false, + }) + } + + /// Sends a request and reads its response, bounded by `deadline`, + /// retrying once with credentials on a 401 challenge. Non-2xx statuses + /// become [`RtspVideoSourceError::RtspStatus`]. + pub(super) fn request( + &mut self, + method: &str, + uri: &str, + headers: &[(&str, &str)], + deadline: Instant, + phase: RtspPhase, + ) -> Result { + self.write_request(method, uri, headers)?; + let mut response = self.read_response(deadline, phase)?; + if response.status_code == 401 { + self.auth.update_from_unauthorized(&response)?; + self.write_request(method, uri, headers)?; + response = self.read_response(deadline, phase)?; + } + + if !response.is_success() { + return Err(RtspVideoSourceError::RtspStatus { + code: response.status_code, + reason: response.reason, + }); + } + Ok(response) + } + + /// Writes a request as a single buffered write, without waiting for the + /// response. Used for keepalives and TEARDOWN, whose replies (if any) + /// arrive in-band. + pub(super) fn write_request( + &mut self, + method: &str, + uri: &str, + extra_headers: &[(&str, &str)], + ) -> Result<(), RtspVideoSourceError> { + let cseq = self.cseq; + self.cseq = self.cseq.saturating_add(1); + let authorization = self.auth.header(method, uri)?; + let request_uri = Url::parse(uri) + .map_err(|_| RtspVideoSourceError::InvalidUrl("request URI is not a valid URL"))?; + + let mut request = rtsp_types::Request::builder(request_method(method), Version::V1_0) + .request_uri(request_uri) + .header(headers::CSEQ, cseq.to_string()) + .header(headers::USER_AGENT, "livekit-capture/0.1".to_owned()); + if let Some(authorization) = authorization { + request = request.header(headers::AUTHORIZATION, authorization); + } + for (name, value) in extra_headers { + // Header names come from this crate's own call sites. + let name = HeaderName::try_from(*name).expect("static header names are valid"); + request = request.header(name, (*value).to_owned()); + } + + let mut bytes = Vec::with_capacity(256); + request + .empty() + .write(&mut bytes) + .map_err(|err| RtspVideoSourceError::Io(io::Error::other(err)))?; + self.stream.write_all(&bytes)?; + self.stream.flush()?; + Ok(()) + } + + /// Reads one RTSP response from the stream, bounded by `deadline`. + fn read_response( + &mut self, + deadline: Instant, + phase: RtspPhase, + ) -> Result { + loop { + if !self.buf.is_empty() { + match Message::parse(&self.buf) { + Ok((message, consumed)) => { + self.buf.advance(consumed); + match message { + Message::Response(response) => { + return Ok(RtspResponse::from_message(response)); + } + Message::Data(_) | Message::Request(_) => { + return Err(RtspVideoSourceError::InvalidResponse( + "expected a response", + )); + } + } + } + Err(ParseError::Incomplete(_)) => self.check_pending_size()?, + Err(ParseError::Error) => { + return Err(RtspVideoSourceError::InvalidResponse("malformed response")); + } + } + } + match self.fill()? { + StreamFill::Filled => {} + StreamFill::Eof => { + return Err(io::Error::from(io::ErrorKind::UnexpectedEof).into()); + } + StreamFill::TimedOut => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::Timeout { phase }); + } + } + } + } + } + + /// Fails when a still-incomplete message exceeds the framing bound. + fn check_pending_size(&self) -> Result<(), RtspVideoSourceError> { + if self.buf.len() > MAX_PENDING_MESSAGE_BYTES { + return Err(RtspVideoSourceError::InvalidResponse("message too large")); + } + Ok(()) + } + + /// Reads the next interleaved unit, returning within roughly one + /// [`READ_POLL`] when the stream is silent. Framing state survives + /// timed-out reads. + pub(super) fn poll_unit(&mut self) -> Result { + loop { + if let Some(unit) = self.parse_front()? { + return Ok(unit); + } + match self.fill()? { + StreamFill::Filled => {} + StreamFill::Eof => { + if self.buf.is_empty() { + return Ok(InterleavedPoll::EndOfStream); + } + // The stream ended inside an interleaved unit. + return Err(io::Error::from(io::ErrorKind::UnexpectedEof).into()); + } + StreamFill::TimedOut => return Ok(InterleavedPoll::TimedOut), + } + } + } + + /// Time since stream bytes last arrived, for the idle limit. + pub(super) fn idle_for(&self) -> Duration { + self.last_read_at.elapsed() + } + + /// Parses one complete unit from the front of the buffer. + fn parse_front(&mut self) -> Result, RtspVideoSourceError> { + loop { + if self.buf.is_empty() { + return Ok(None); + } + match Message::parse(&self.buf) { + Ok((message, consumed)) => { + self.buf.advance(consumed); + match message { + Message::Data(data) => { + let channel = data.channel_id(); + let payload = Bytes::from(data.into_body()); + return Ok(Some(InterleavedPoll::Frame { channel, payload })); + } + Message::Response(response) => { + return Ok(Some(InterleavedPoll::Response( + RtspResponse::from_message(response), + ))); + } + Message::Request(request) => { + // Some servers send requests (ANNOUNCE, keepalive + // checks) to the client mid-stream; ignore them. + if !self.logged_server_request { + self.logged_server_request = true; + log::warn!( + "ignoring in-band RTSP {:?} request from the server", + request.method(), + ); + } + } + } + } + Err(ParseError::Incomplete(_)) => { + self.check_pending_size()?; + return Ok(None); + } + Err(ParseError::Error) => return Err(RtspVideoSourceError::UnexpectedData), + } + } + } + + /// Reads more stream bytes into the buffer. + fn fill(&mut self) -> Result { + loop { + match self.stream.read(&mut self.scratch) { + Ok(0) => return Ok(StreamFill::Eof), + Ok(read) => { + self.buf.extend_from_slice(&self.scratch[..read]); + self.last_read_at = Instant::now(); + return Ok(StreamFill::Filled); + } + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) if is_timeout_io_error(&err) => return Ok(StreamFill::TimedOut), + // TLS peers that drop the connection without a close_notify + // (most cameras) surface as UnexpectedEof; treat it like a + // plain EOF and let the caller decide whether the framing + // was left mid-unit. + Err(err) if err.kind() == io::ErrorKind::UnexpectedEof => { + return Ok(StreamFill::Eof) + } + Err(err) => return Err(err.into()), + } + } + } +} + +/// TLS support for `rtsps://` URLs. +#[cfg(feature = "source-rtsp-tls")] +mod tls { + use std::{io, net::TcpStream, sync::Arc, time::Instant}; + + use rustls::{ClientConfig, ClientConnection, RootCertStore, StreamOwned}; + use rustls_pki_types::ServerName; + + use super::{is_timeout_io_error, sanitized, RtspPhase, RtspVideoSourceError}; + + /// Establishes TLS over a connected TCP stream, driving the handshake to + /// completion bounded by `deadline`. + /// + /// The handshake must finish here: afterwards, writes never need to + /// read, so the request path's timeout handling stays valid. During the + /// handshake, the socket's read timeout is the retry granularity. + pub(super) fn establish( + stream: TcpStream, + host: &str, + accept_invalid_certs: bool, + deadline: Instant, + ) -> Result, RtspVideoSourceError> { + let config = client_config(accept_invalid_certs)?; + // `ServerName` accepts both DNS names and the IP literals cameras + // are usually addressed by. + let server_name = ServerName::try_from(host.to_owned()) + .map_err(|err| RtspVideoSourceError::Tls(sanitized(err.to_string())))?; + let mut connection = ClientConnection::new(Arc::new(config), server_name) + .map_err(|err| RtspVideoSourceError::Tls(sanitized(err.to_string())))?; + + let mut stream = stream; + while connection.is_handshaking() { + match connection.complete_io(&mut stream) { + Ok(_) => {} + Err(err) if err.kind() == io::ErrorKind::Interrupted => {} + Err(err) if is_timeout_io_error(&err) => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::Timeout { phase: RtspPhase::Connect }); + } + } + // rustls reports TLS-level handshake failures as InvalidData. + Err(err) if err.kind() == io::ErrorKind::InvalidData => { + return Err(RtspVideoSourceError::Tls(sanitized(err.to_string()))); + } + Err(err) => return Err(err.into()), + } + } + Ok(StreamOwned::new(connection, stream)) + } + + fn client_config(accept_invalid_certs: bool) -> Result { + if accept_invalid_certs { + return Ok(ClientConfig::builder() + .dangerous() + .with_custom_certificate_verifier(Arc::new(danger::NoVerification)) + .with_no_client_auth()); + } + + let mut roots = RootCertStore::empty(); + // Individually unparsable certificates in the OS store are skipped; + // an entirely unavailable store is an error. + let native = rustls_native_certs::load_native_certs(); + for cert in native.certs { + let _ = roots.add(cert); + } + if roots.is_empty() { + return Err(RtspVideoSourceError::Tls( + "no usable system root certificates".to_owned(), + )); + } + Ok(ClientConfig::builder().with_root_certificates(roots).with_no_client_auth()) + } + + /// Certificate "verification" that accepts anything; see + /// [`RtspVideoSourceConfig::accept_invalid_tls_certs`](super::super::RtspVideoSourceConfig::accept_invalid_tls_certs). + mod danger { + use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}, + crypto::{ring, verify_tls12_signature, verify_tls13_signature}, + DigitallySignedStruct, Error, SignatureScheme, + }; + use rustls_pki_types::{CertificateDer, ServerName, UnixTime}; + + #[derive(Debug)] + pub(super) struct NoVerification; + + impl ServerCertVerifier for NoVerification { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer<'_>, + _intermediates: &[CertificateDer<'_>], + _server_name: &ServerName<'_>, + _ocsp_response: &[u8], + _now: UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls12_signature( + message, + cert, + dss, + &ring::default_provider().signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + verify_tls13_signature( + message, + cert, + dss, + &ring::default_provider().signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + ring::default_provider().signature_verification_algorithms.supported_schemes() + } + } + } +} + +fn is_timeout_io_error(err: &io::Error) -> bool { + matches!(err.kind(), io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut) +} + +/// Maps a request method from this crate's own call sites to its typed form. +fn request_method(method: &str) -> Method { + match method { + "DESCRIBE" => Method::Describe, + "SETUP" => Method::Setup, + "PLAY" => Method::Play, + "OPTIONS" => Method::Options, + "TEARDOWN" => Method::Teardown, + other => Method::Extension(other.to_owned()), + } +} + +#[cfg(test)] +impl RtspResponse { + /// Parses one complete response, for tests in sibling modules. + pub(super) fn parse_for_tests(bytes: &[u8]) -> Self { + match Message::parse(bytes).expect("invalid response") { + (Message::Response(response), _) => Self::from_message(response), + (other, _) => panic!("expected a response, got {other:?}"), + } + } +} + +/// Extracts the session identifier from a `Session` header value. +pub(super) fn parse_session_id(session_header: &str) -> Result { + let session_id = session_header.split(';').next().unwrap_or_default().trim(); + if session_id.is_empty() { + return Err(RtspVideoSourceError::InvalidResponse("empty session id")); + } + Ok(session_id.to_owned()) +} + +/// Extracts the `timeout` parameter of a `Session` header value. +pub(super) fn parse_session_timeout_secs(session_header: &str) -> Option { + session_header.split(';').skip(1).find_map(|part| { + let (name, value) = part.trim().split_once('=')?; + if name.trim().eq_ignore_ascii_case("timeout") { + value.trim().parse().ok() + } else { + None + } + }) +} + +/// Extracts the RTP channel from a SETUP response's `Transport` header. +pub(super) fn parse_interleaved_channel( + transport_header: Option<&str>, +) -> Result { + let transport_header = + transport_header.ok_or(RtspVideoSourceError::MissingHeader("Transport"))?; + for part in transport_header.split(';') { + let Some(value) = part.trim().strip_prefix("interleaved=") else { + continue; + }; + if let Some(first) = value.split('-').next().and_then(|channel| channel.parse().ok()) { + return Ok(first); + } + } + Err(RtspVideoSourceError::InvalidResponse("Transport header without interleaved channels")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_credentials_but_strips_them_from_request_uri() { + let url = RtspUrl::parse("rtsp://admin:secret@camera.example:554/live").unwrap(); + + assert_eq!(url.request_uri, "rtsp://camera.example:554/live"); + assert_eq!( + url.credentials, + Some(RtspCredentials { username: "admin".to_owned(), password: "secret".to_owned() }) + ); + } + + #[test] + fn percent_decodes_userinfo() { + let url = RtspUrl::parse("rtsp://user%40lk:p%40ss%2Fword@camera.example/live").unwrap(); + assert_eq!( + url.credentials, + Some(RtspCredentials { + username: "user@lk".to_owned(), + password: "p@ss/word".to_owned(), + }) + ); + } + + #[test] + fn parses_rtsps_urls() { + let url = RtspUrl::parse("rtsps://camera.example/live").unwrap(); + assert!(url.tls); + assert_eq!(url.port, 322); + assert_eq!(url.request_uri, "rtsps://camera.example/live"); + + let url = RtspUrl::parse("rtsps://admin:secret@camera.example:7441/live").unwrap(); + assert!(url.tls); + assert_eq!(url.port, 7441); + assert_eq!(url.request_uri, "rtsps://camera.example:7441/live"); + assert!(url.credentials.is_some()); + + assert!(!RtspUrl::parse("rtsp://camera.example/live").unwrap().tls); + } + + #[test] + fn defaults_to_port_554() { + let url = RtspUrl::parse("rtsp://camera.example/live").unwrap(); + assert_eq!(url.port, 554); + assert_eq!(url.request_uri, "rtsp://camera.example/live"); + } + + #[test] + fn parses_bracketed_ipv6_host() { + let url = RtspUrl::parse("rtsp://[2001:db8::1]:8554/live").unwrap(); + assert_eq!(url.connect_host, "2001:db8::1"); + assert_eq!(url.port, 8554); + assert_eq!(url.request_uri, "rtsp://[2001:db8::1]:8554/live"); + } + + #[test] + fn rejects_invalid_urls() { + assert!(matches!( + RtspUrl::parse("http://camera.example/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp:///live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp://:secret@camera.example/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + assert!(matches!( + RtspUrl::parse("rtsp://camera.example:notaport/live"), + Err(RtspVideoSourceError::InvalidUrl(_)) + )); + } + + #[test] + fn rejects_oversized_incomplete_message() { + use std::{net::TcpListener, thread}; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + // Claim a body far larger than the framing bound, stream past + // the bound without completing it, then stall. + stream + .write_all(b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 999999\r\n\r\n") + .unwrap(); + stream.write_all(&vec![b'a'; MAX_PENDING_MESSAGE_BYTES + 8 * 1024]).unwrap(); + stream.flush().unwrap(); + thread::sleep(Duration::from_millis(200)); + }); + + let url = RtspUrl::parse(&format!("rtsp://{addr}/test")).unwrap(); + let deadline = Instant::now() + Duration::from_secs(2); + let mut client = RtspClient::connect(&url, None, false, deadline).unwrap(); + let err = client + .request("DESCRIBE", &url.request_uri, &[], deadline, RtspPhase::Describe) + .unwrap_err(); + + assert!( + matches!(err, RtspVideoSourceError::InvalidResponse("message too large")), + "unexpected error: {err:?}" + ); + server.join().unwrap(); + } + + #[test] + fn converts_parsed_responses() { + let response = RtspResponse::parse_for_tests( + b"RTSP/1.0 200 OK\r\nCSeq: 1\r\nContent-Length: 4\r\n\r\nbody", + ); + + assert_eq!(response.status_code, 200); + assert_eq!(response.reason, "OK"); + assert_eq!(response.header("cseq"), Some("1")); + assert_eq!(response.body, b"body"); + assert!(response.is_success()); + } + + #[test] + fn strips_control_characters_from_reason_phrases() { + let message = rtsp_types::Response::builder(Version::V1_0, rtsp_types::StatusCode::Ok) + .reason_phrase("OK\u{1b}[31m\nfake log line") + .build(Vec::new()); + + let response = RtspResponse::from_message(message); + assert_eq!(response.reason, "OK[31mfake log line"); + } + + #[test] + fn parses_session_header() { + assert_eq!(parse_session_id("abc123;timeout=60").unwrap(), "abc123"); + assert!(parse_session_id(" ;timeout=60").is_err()); + assert_eq!(parse_session_timeout_secs("abc123;timeout=60"), Some(60)); + assert_eq!(parse_session_timeout_secs("abc123; Timeout = 30"), Some(30)); + assert_eq!(parse_session_timeout_secs("abc123"), None); + } + + #[test] + fn parses_interleaved_channel() { + assert_eq!( + parse_interleaved_channel(Some("RTP/AVP/TCP;unicast;interleaved=2-3")).unwrap(), + 2 + ); + assert!(matches!( + parse_interleaved_channel(Some("RTP/AVP/TCP;unicast")), + Err(RtspVideoSourceError::InvalidResponse(_)) + )); + assert!(matches!( + parse_interleaved_channel(None), + Err(RtspVideoSourceError::MissingHeader("Transport")) + )); + } +} diff --git a/livekit-capture/src/sources/rtsp/dimensions.rs b/livekit-capture/src/sources/rtsp/dimensions.rs new file mode 100644 index 000000000..b39ce98fe --- /dev/null +++ b/livekit-capture/src/sources/rtsp/dimensions.rs @@ -0,0 +1,635 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Frame dimension parsing from keyframe payloads, for resolution discovery. +//! +//! Every parser returns `None` on malformed or unexpected input; discovery +//! treats that as "not discoverable" rather than a stream error. + +use super::bits::{BitReader, ByteReader}; +use crate::{ + encoded::{ + h26x::{annex_b_nalus, h264_nal_type, h265_nal_type, H264_NAL_SPS, H265_NAL_SPS}, + EncodedVideoCodec, + }, + primitive::VideoResolution, +}; + +/// Extracts the frame dimensions from a keyframe access-unit payload. +pub(super) fn access_unit_resolution( + codec: EncodedVideoCodec, + payload: &[u8], +) -> Option { + match codec { + EncodedVideoCodec::H264 => annex_b_nalus(payload) + .into_iter() + .find(|nal| matches!(h264_nal_type(nal), Ok(H264_NAL_SPS))) + .and_then(|nal| sps_resolution(codec, nal)), + EncodedVideoCodec::H265 => annex_b_nalus(payload) + .into_iter() + .find(|nal| matches!(h265_nal_type(nal), Ok(H265_NAL_SPS))) + .and_then(|nal| sps_resolution(codec, nal)), + EncodedVideoCodec::VP8 => vp8_keyframe_resolution(payload), + EncodedVideoCodec::VP9 => vp9_keyframe_resolution(payload), + EncodedVideoCodec::AV1 => av1_sequence_header_resolution(payload), + } +} + +/// Extracts the frame dimensions from a raw H.264 or H.265 SPS NAL unit, +/// such as one carried out-of-band in SDP `sprop` attributes. +pub(super) fn sps_resolution( + codec: EncodedVideoCodec, + sps_nal: &[u8], +) -> Option { + match codec { + EncodedVideoCodec::H264 => h264_sps_resolution(&rbsp_from_nal(sps_nal, 1)?), + EncodedVideoCodec::H265 => h265_sps_resolution(&rbsp_from_nal(sps_nal, 2)?), + _ => None, + } +} + +/// Strips the NAL header and emulation-prevention bytes (`00 00 03`), which +/// must not reach the bit-level parsers. +fn rbsp_from_nal(nal: &[u8], header_len: usize) -> Option> { + let payload = nal.get(header_len..)?; + let mut rbsp = Vec::with_capacity(payload.len()); + let mut zeros = 0usize; + for &byte in payload { + if zeros >= 2 && byte == 0x03 { + zeros = 0; + continue; + } + zeros = if byte == 0 { zeros + 1 } else { 0 }; + rbsp.push(byte); + } + Some(rbsp) +} + +/// Parses the dimensions from an H.264 SPS RBSP (ITU-T H.264 section 7.3.2.1). +fn h264_sps_resolution(rbsp: &[u8]) -> Option { + let mut reader = BitReader::new(rbsp); + let profile_idc = reader.read_bits(8)?; + reader.skip_bits(8)?; // constraint flags + reserved + reader.skip_bits(8)?; // level_idc + reader.read_ue()?; // seq_parameter_set_id + + let mut chroma_format_idc = 1; + let mut separate_colour_plane = false; + if matches!(profile_idc, 100 | 110 | 122 | 244 | 44 | 83 | 86 | 118 | 128 | 138 | 139 | 134 | 135) + { + chroma_format_idc = reader.read_ue()?; + if chroma_format_idc == 3 { + separate_colour_plane = reader.read_flag()?; + } + reader.read_ue()?; // bit_depth_luma_minus8 + reader.read_ue()?; // bit_depth_chroma_minus8 + reader.read_bit()?; // qpprime_y_zero_transform_bypass_flag + if reader.read_flag()? { + // seq_scaling_matrix_present_flag + let lists = if chroma_format_idc == 3 { 12 } else { 8 }; + for index in 0..lists { + if reader.read_flag()? { + skip_h264_scaling_list(&mut reader, if index < 6 { 16 } else { 64 })?; + } + } + } + } + + reader.read_ue()?; // log2_max_frame_num_minus4 + let pic_order_cnt_type = reader.read_ue()?; + if pic_order_cnt_type == 0 { + reader.read_ue()?; // log2_max_pic_order_cnt_lsb_minus4 + } else if pic_order_cnt_type == 1 { + reader.read_bit()?; // delta_pic_order_always_zero_flag + reader.skip_se()?; // offset_for_non_ref_pic + reader.skip_se()?; // offset_for_top_to_bottom_field + let cycle_length = reader.read_ue()?; + for _ in 0..cycle_length { + reader.skip_se()?; // offset_for_ref_frame + } + } + reader.read_ue()?; // max_num_ref_frames + reader.read_bit()?; // gaps_in_frame_num_value_allowed_flag + + let pic_width_in_mbs = reader.read_ue()?.checked_add(1)?; + let pic_height_in_map_units = reader.read_ue()?.checked_add(1)?; + let frame_mbs_only = reader.read_flag()?; + if !frame_mbs_only { + reader.read_bit()?; // mb_adaptive_frame_field_flag + } + reader.read_bit()?; // direct_8x8_inference_flag + + let frame_height_factor = if frame_mbs_only { 1 } else { 2 }; + let mut width = pic_width_in_mbs.checked_mul(16)?; + let mut height = + pic_height_in_map_units.checked_mul(16)?.checked_mul(frame_height_factor)?; + + if reader.read_flag()? { + // frame_cropping_flag + let crop_left = reader.read_ue()?; + let crop_right = reader.read_ue()?; + let crop_top = reader.read_ue()?; + let crop_bottom = reader.read_ue()?; + let chroma_array_type = if separate_colour_plane { 0 } else { chroma_format_idc }; + let (sub_width, sub_height) = match chroma_array_type { + 0 => (1, 1), + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + let crop_unit_x = sub_width; + let crop_unit_y = sub_height * frame_height_factor; + width = width.checked_sub(crop_left.checked_add(crop_right)?.checked_mul(crop_unit_x)?)?; + height = + height.checked_sub(crop_top.checked_add(crop_bottom)?.checked_mul(crop_unit_y)?)?; + } + + checked_resolution(width, height) +} + +fn skip_h264_scaling_list(reader: &mut BitReader<'_>, size: u32) -> Option<()> { + let mut last_scale = 8i32; + let mut next_scale = 8i32; + for _ in 0..size { + if next_scale != 0 { + let delta = read_se(reader)?; + next_scale = (last_scale + delta).rem_euclid(256); + } + if next_scale != 0 { + last_scale = next_scale; + } + } + Some(()) +} + +fn read_se(reader: &mut BitReader<'_>) -> Option { + let code = reader.read_ue()?; + let magnitude = code.div_ceil(2) as i32; + Some(if code % 2 == 1 { magnitude } else { -magnitude }) +} + +/// Parses the dimensions from an H.265 SPS RBSP (ITU-T H.265 section 7.3.2.2). +fn h265_sps_resolution(rbsp: &[u8]) -> Option { + let mut reader = BitReader::new(rbsp); + reader.skip_bits(4)?; // sps_video_parameter_set_id + let max_sub_layers_minus1 = reader.read_bits(3)? as usize; + reader.read_bit()?; // sps_temporal_id_nesting_flag + skip_h265_profile_tier_level(&mut reader, max_sub_layers_minus1)?; + reader.read_ue()?; // sps_seq_parameter_set_id + + let chroma_format_idc = reader.read_ue()?; + if chroma_format_idc == 3 { + reader.read_bit()?; // separate_colour_plane_flag + } + let mut width = reader.read_ue()?; + let mut height = reader.read_ue()?; + + if reader.read_flag()? { + // conformance_window_flag + let left = reader.read_ue()?; + let right = reader.read_ue()?; + let top = reader.read_ue()?; + let bottom = reader.read_ue()?; + let (sub_width, sub_height) = match chroma_format_idc { + 1 => (2, 2), + 2 => (2, 1), + _ => (1, 1), + }; + width = width.checked_sub(left.checked_add(right)?.checked_mul(sub_width)?)?; + height = height.checked_sub(top.checked_add(bottom)?.checked_mul(sub_height)?)?; + } + + checked_resolution(width, height) +} + +/// Skips a `profile_tier_level` structure with `profilePresentFlag = 1`. +fn skip_h265_profile_tier_level( + reader: &mut BitReader<'_>, + max_sub_layers_minus1: usize, +) -> Option<()> { + reader.skip_bits(88)?; // general profile space/tier/idc/compat/constraints + reader.skip_bits(8)?; // general_level_idc + + let mut profile_present = [false; 8]; + let mut level_present = [false; 8]; + for index in 0..max_sub_layers_minus1.min(8) { + profile_present[index] = reader.read_flag()?; + level_present[index] = reader.read_flag()?; + } + if max_sub_layers_minus1 > 0 { + for _ in max_sub_layers_minus1..8 { + reader.skip_bits(2)?; // reserved_zero_2bits + } + } + for index in 0..max_sub_layers_minus1.min(8) { + if profile_present[index] { + reader.skip_bits(88)?; + } + if level_present[index] { + reader.skip_bits(8)?; + } + } + Some(()) +} + +/// Parses the dimensions from a VP8 keyframe payload (RFC 6386 section 9.1). +fn vp8_keyframe_resolution(payload: &[u8]) -> Option { + let header = *payload.first()?; + if header & 0x01 != 0 { + return None; // not a keyframe + } + if payload.get(3..6)? != [0x9d, 0x01, 0x2a] { + return None; // missing keyframe start code + } + let width = u32::from(u16::from_le_bytes([*payload.get(6)?, *payload.get(7)?]) & 0x3fff); + let height = u32::from(u16::from_le_bytes([*payload.get(8)?, *payload.get(9)?]) & 0x3fff); + checked_resolution(width, height) +} + +/// Parses the dimensions from a VP9 keyframe's uncompressed header. +fn vp9_keyframe_resolution(payload: &[u8]) -> Option { + let mut reader = BitReader::new(payload); + if reader.read_bits(2)? != 0b10 { + return None; // frame_marker + } + let profile = reader.read_bit()? | (reader.read_bit()? << 1); + if profile == 3 { + reader.read_bit()?; // reserved_zero + } + if reader.read_flag()? { + return None; // show_existing_frame repeats a decoded frame + } + if reader.read_bit()? != 0 { + return None; // frame_type: not a keyframe + } + reader.read_bit()?; // show_frame + reader.read_bit()?; // error_resilient_mode + if reader.read_bits(24)? != 0x49_83_42 { + return None; // frame_sync_code + } + + // color_config + if profile >= 2 { + reader.read_bit()?; // ten_or_twelve_bit + } + let color_space = reader.read_bits(3)?; + const CS_RGB: u32 = 7; + if color_space != CS_RGB { + reader.read_bit()?; // color_range + if profile == 1 || profile == 3 { + reader.skip_bits(3)?; // subsampling_x, subsampling_y, reserved + } + } else if profile == 1 || profile == 3 { + reader.read_bit()?; // reserved_zero + } + + let width = reader.read_bits(16)?.checked_add(1)?; + let height = reader.read_bits(16)?.checked_add(1)?; + checked_resolution(width, height) +} + +/// Parses the maximum frame dimensions from the sequence header OBU of an +/// AV1 access unit built from size-prefixed OBUs. +fn av1_sequence_header_resolution(payload: &[u8]) -> Option { + let mut reader = ByteReader::new(payload); + while !reader.is_empty() { + let header = reader.get_u8()?; + if header & 0x80 != 0 { + return None; // obu_forbidden_bit + } + let obu_type = (header & 0x78) >> 3; + let has_extension = header & 0x04 != 0; + let has_size = header & 0x02 != 0; + if has_extension { + reader.skip(1)?; + } + if !has_size { + // Without a size field the OBU extends to the end of the unit. + return (obu_type == 1) + .then(|| av1_sequence_header_obu_resolution(reader.take_rest())) + .flatten(); + } + let size = reader.get_leb128()?; + let obu_payload = reader.take(size)?; + if obu_type == 1 { + return av1_sequence_header_obu_resolution(obu_payload); + } + } + None +} + +/// Parses `max_frame_width/height` from a sequence header OBU payload +/// (AV1 specification section 5.5.1). +fn av1_sequence_header_obu_resolution(payload: &[u8]) -> Option { + let mut reader = BitReader::new(payload); + reader.read_bits(3)?; // seq_profile + reader.read_bit()?; // still_picture + let reduced_still_picture_header = reader.read_flag()?; + if reduced_still_picture_header { + reader.skip_bits(5)?; // seq_level_idx[0] + } else { + let mut decoder_model_info_present = false; + let mut buffer_delay_length = 0usize; + if reader.read_flag()? { + // timing_info_present_flag + reader.skip_bits(32)?; // num_units_in_display_tick + reader.skip_bits(32)?; // time_scale + if reader.read_flag()? { + // equal_picture_interval + reader.read_ue()?; // num_ticks_per_picture_minus_1 (uvlc) + } + decoder_model_info_present = reader.read_flag()?; + if decoder_model_info_present { + buffer_delay_length = reader.read_bits(5)? as usize + 1; + reader.skip_bits(32)?; // num_units_in_decoding_tick + reader.skip_bits(10)?; // buffer_removal + frame_presentation lengths + } + } + let initial_display_delay_present = reader.read_flag()?; + let operating_points = reader.read_bits(5)? as usize + 1; + for _ in 0..operating_points { + reader.skip_bits(12)?; // operating_point_idc + let seq_level_idx = reader.read_bits(5)?; + if seq_level_idx > 7 { + reader.read_bit()?; // seq_tier + } + if decoder_model_info_present && reader.read_flag()? { + reader.skip_bits(buffer_delay_length * 2 + 1)?; // operating_parameters_info + } + if initial_display_delay_present && reader.read_flag()? { + reader.skip_bits(4)?; // initial_display_delay_minus_1 + } + } + } + + let width_bits = reader.read_bits(4)? + 1; + let height_bits = reader.read_bits(4)? + 1; + let width = reader.read_bits(width_bits)?.checked_add(1)?; + let height = reader.read_bits(height_bits)?.checked_add(1)?; + checked_resolution(width, height) +} + +fn checked_resolution(width: u32, height: u32) -> Option { + (width > 0 && height > 0).then_some(VideoResolution::new(width, height)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// MSB-first bit writer for composing test bitstreams. + #[derive(Default)] + struct BitWriter { + bytes: Vec, + bit_offset: usize, + } + + impl BitWriter { + fn push_bit(&mut self, bit: u32) { + if self.bit_offset % 8 == 0 { + self.bytes.push(0); + } + let byte = self.bytes.last_mut().unwrap(); + *byte |= ((bit & 1) as u8) << (7 - self.bit_offset % 8); + self.bit_offset += 1; + } + + fn push_bits(&mut self, value: u32, bits: u32) { + for offset in (0..bits).rev() { + self.push_bit((value >> offset) & 1); + } + } + + fn push_ue(&mut self, value: u32) { + let code = value + 1; + let bits = 32 - code.leading_zeros(); + self.push_bits(0, bits - 1); + self.push_bits(code, bits); + } + + fn finish(mut self) -> Vec { + // rbsp_stop_one_bit and alignment. + self.push_bit(1); + while self.bit_offset % 8 != 0 { + self.push_bit(0); + } + self.bytes + } + } + + fn h264_sps_nal(profile_idc: u32, width: u32, height: u32, crop_bottom: u32) -> Vec { + let mut writer = BitWriter::default(); + writer.push_bits(profile_idc, 8); + writer.push_bits(0, 8); // constraint flags + writer.push_bits(30, 8); // level_idc + writer.push_ue(0); // seq_parameter_set_id + if profile_idc == 100 { + writer.push_ue(1); // chroma_format_idc (4:2:0) + writer.push_ue(0); // bit_depth_luma_minus8 + writer.push_ue(0); // bit_depth_chroma_minus8 + writer.push_bit(0); // qpprime_y_zero_transform_bypass_flag + writer.push_bit(0); // seq_scaling_matrix_present_flag + } + writer.push_ue(0); // log2_max_frame_num_minus4 + writer.push_ue(0); // pic_order_cnt_type + writer.push_ue(0); // log2_max_pic_order_cnt_lsb_minus4 + writer.push_ue(1); // max_num_ref_frames + writer.push_bit(0); // gaps_in_frame_num_value_allowed_flag + writer.push_ue(width / 16 - 1); + writer.push_ue(height.div_ceil(16) - 1); + writer.push_bit(1); // frame_mbs_only_flag + writer.push_bit(1); // direct_8x8_inference_flag + if crop_bottom > 0 { + writer.push_bit(1); // frame_cropping_flag + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(crop_bottom / 2); // CropUnitY = 2 for 4:2:0 + } else { + writer.push_bit(0); + } + writer.push_bit(0); // vui_parameters_present_flag + + let mut nal = vec![0x67]; + nal.extend(writer.finish()); + nal + } + + fn h265_sps_nal(width: u32, height: u32, crop_bottom: u32) -> Vec { + let mut writer = BitWriter::default(); + writer.push_bits(0, 4); // sps_video_parameter_set_id + writer.push_bits(0, 3); // sps_max_sub_layers_minus1 + writer.push_bit(1); // sps_temporal_id_nesting_flag + writer.push_bits(0, 32); // profile_tier_level: space/tier/idc/compat... + writer.push_bits(0, 32); + writer.push_bits(0, 24); // ...constraint and reserved bits (88 total) + writer.push_bits(93, 8); // general_level_idc + writer.push_ue(0); // sps_seq_parameter_set_id + writer.push_ue(1); // chroma_format_idc (4:2:0) + writer.push_ue(width); + writer.push_ue(height + crop_bottom * 2); + if crop_bottom > 0 { + writer.push_bit(1); // conformance_window_flag + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(0); + writer.push_ue(crop_bottom); // SubHeightC = 2 for 4:2:0 + } else { + writer.push_bit(0); + } + + let mut nal = vec![0x42, 0x01]; + nal.extend(writer.finish()); + nal + } + + #[test] + fn parses_h264_sps_dimensions() { + let nal = h264_sps_nal(66, 640, 480, 0); + assert_eq!( + sps_resolution(EncodedVideoCodec::H264, &nal), + Some(VideoResolution::new(640, 480)) + ); + } + + #[test] + fn parses_h264_high_profile_sps_with_cropping() { + // 1920x1088 coded, cropped to 1920x1080. + let nal = h264_sps_nal(100, 1920, 1080, 8); + assert_eq!( + sps_resolution(EncodedVideoCodec::H264, &nal), + Some(VideoResolution::new(1920, 1080)) + ); + } + + #[test] + fn parses_h264_sps_from_access_unit() { + let sps = h264_sps_nal(66, 1280, 720, 0); + let mut payload = Vec::new(); + payload.extend_from_slice(&[0, 0, 0, 1]); + payload.extend_from_slice(&sps); + payload.extend_from_slice(&[0, 0, 0, 1, 0x68, 0x08]); + payload.extend_from_slice(&[0, 0, 0, 1, 0x65, 0x88, 0x84]); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::H264, &payload), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_h265_sps_dimensions() { + let nal = h265_sps_nal(1280, 720, 0); + assert_eq!( + sps_resolution(EncodedVideoCodec::H265, &nal), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_h265_sps_with_conformance_window() { + // 1920x1088 coded, cropped to 1920x1080. + let nal = h265_sps_nal(1920, 1080, 4); + assert_eq!( + sps_resolution(EncodedVideoCodec::H265, &nal), + Some(VideoResolution::new(1920, 1080)) + ); + } + + #[test] + fn strips_emulation_prevention_bytes() { + assert_eq!(rbsp_from_nal(&[0x67, 0x01, 0x00, 0x00, 0x03, 0x02], 1), Some(vec![ + 0x01, 0x00, 0x00, 0x02 + ])); + // The escape only applies after two zero bytes. + assert_eq!(rbsp_from_nal(&[0x67, 0x01, 0x00, 0x03, 0x02], 1), Some(vec![ + 0x01, 0x00, 0x03, 0x02 + ])); + } + + #[test] + fn parses_vp8_keyframe_dimensions() { + let payload = [ + 0x00, 0x00, 0x00, // frame tag: keyframe + 0x9d, 0x01, 0x2a, // start code + 0x80, 0x02, // width 640 + 0xe0, 0x01, // height 480 + ]; + assert_eq!( + access_unit_resolution(EncodedVideoCodec::VP8, &payload), + Some(VideoResolution::new(640, 480)) + ); + } + + #[test] + fn rejects_vp8_delta_frame() { + let payload = [0x01, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01]; + assert_eq!(access_unit_resolution(EncodedVideoCodec::VP8, &payload), None); + } + + #[test] + fn parses_vp9_keyframe_dimensions() { + let mut writer = BitWriter::default(); + writer.push_bits(0b10, 2); // frame_marker + writer.push_bits(0, 2); // profile 0 + writer.push_bit(0); // show_existing_frame + writer.push_bit(0); // frame_type: keyframe + writer.push_bit(1); // show_frame + writer.push_bit(0); // error_resilient_mode + writer.push_bits(0x49_83_42, 24); // frame_sync_code + writer.push_bits(0, 3); // color_space + writer.push_bit(0); // color_range + writer.push_bits(1280 - 1, 16); + writer.push_bits(720 - 1, 16); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::VP9, &writer.finish()), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn parses_av1_sequence_header_dimensions() { + let mut writer = BitWriter::default(); + writer.push_bits(0, 3); // seq_profile + writer.push_bit(0); // still_picture + writer.push_bit(0); // reduced_still_picture_header + writer.push_bit(0); // timing_info_present_flag + writer.push_bit(0); // initial_display_delay_present_flag + writer.push_bits(0, 5); // operating_points_cnt_minus_1 + writer.push_bits(0, 12); // operating_point_idc[0] + writer.push_bits(5, 5); // seq_level_idx[0] + writer.push_bits(10, 4); // frame_width_bits_minus_1 + writer.push_bits(9, 4); // frame_height_bits_minus_1 + writer.push_bits(1280 - 1, 11); // max_frame_width_minus_1 + writer.push_bits(720 - 1, 10); // max_frame_height_minus_1 + let obu_payload = writer.finish(); + + // Size-prefixed sequence header OBU followed by an unrelated OBU. + let mut payload = vec![0x0a, obu_payload.len() as u8]; + payload.extend(&obu_payload); + payload.extend_from_slice(&[0x32, 0x01, 0x10]); + + assert_eq!( + access_unit_resolution(EncodedVideoCodec::AV1, &payload), + Some(VideoResolution::new(1280, 720)) + ); + } + + #[test] + fn skips_leading_av1_obus_without_sequence_header() { + let payload = [0x12, 0x00, 0x32, 0x01, 0x10]; // temporal delimiter + frame + assert_eq!(access_unit_resolution(EncodedVideoCodec::AV1, &payload), None); + } +} diff --git a/livekit-capture/src/sources/rtsp/mod.rs b/livekit-capture/src/sources/rtsp/mod.rs new file mode 100644 index 000000000..5cb53efe9 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/mod.rs @@ -0,0 +1,1135 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Encoded video capture from an RTSP server. +//! +//! [`RtspVideoSource`] connects to an `rtsp://` URL, negotiates a video +//! stream over TCP-interleaved RTP, and yields its access units without +//! re-encoding. Basic and Digest authentication are supported, and packet +//! loss is recovered by waiting for the next keyframe. +//! +//! With the `source-rtsp-tls` feature, `rtsps://` URLs are supported: the +//! whole connection — control and interleaved media — runs over TLS (1.2 or +//! newer). Certificates are verified against the system roots unless +//! [`RtspVideoSourceConfig::accept_invalid_tls_certs`] opts out. +//! +//! The connection is not re-established on failure: a connection error ends +//! the source with an error, and a clean server-side end of stream ends it +//! like any finite source. +//! +//! One video track is ingested per source: when the SDP offers several, the +//! first one carrying a supported (or the configured) codec is used and the +//! choice is logged. Cameras exposing multiple streams as separate URLs are +//! served by one source per URL. + +mod auth; +mod bits; +mod client; +mod dimensions; +mod rtp; +mod sdp; + +use std::{ + fmt, io, str, + time::{Duration, Instant}, +}; + +use thiserror::Error; + +use crate::{ + encoded::{EncodedFrameType, EncodedVideoCodec, EncodedVideoSource, OwnedEncodedAccessUnit}, + error::SourceError, + primitive::VideoResolution, + pump::PumpStop, +}; +use auth::RtspCredentials; +use client::{ + parse_interleaved_channel, parse_session_id, parse_session_timeout_secs, InterleavedPoll, + RtspClient, RtspUrl, +}; +use rtp::RtpAccessUnitAssembler; + +/// Default TCP connect and handshake timeout. +const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Default maximum stream silence tolerated before the source fails. +const DEFAULT_IDLE_TIMEOUT: Duration = Duration::from_secs(30); + +/// How long resolution discovery waits for the stream's first keyframe. +const DISCOVERY_TIMEOUT: Duration = Duration::from_secs(10); + +/// Session timeout assumed when the server's `Session` header declares none +/// (RFC 2326 section 12.37). +const DEFAULT_SESSION_TIMEOUT_SECS: u64 = 60; + +/// Configuration for an RTSP encoded video source. +#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(deny_unknown_fields) +)] +#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))] +pub struct RtspVideoSourceConfig { + /// RTSP URL (`rtsp://host[:port]/path`, or `rtsps://` with the + /// `source-rtsp-tls` feature). + /// + /// URL userinfo (`rtsp://user:password@...`) is accepted and stripped + /// from requests; [`Self::username`] and [`Self::password`] take + /// precedence over it. + pub url: String, + + /// Username for RTSP authentication, overriding URL userinfo. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub username: Option, + + /// Password for RTSP authentication, overriding URL userinfo. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub password: Option, + + /// Codec required from the stream. When omitted, the first supported + /// video track offered by the SDP is used. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub codec: Option, + + /// Encoded frame resolution. + /// + /// When omitted, the resolution is discovered from the SDP when it + /// declares one, and from the stream's first keyframe otherwise, so + /// construction may wait for the stream to produce data. When set, + /// construction returns without waiting, and the first keyframe is + /// verified against this value. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub resolution: Option, + + /// TCP connect and RTSP handshake timeout in milliseconds + /// (default 10000). + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub connect_timeout_ms: Option, + + /// Maximum stream silence tolerated before the source fails, in + /// milliseconds (default 30000). Receiving any stream bytes resets the + /// limit. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "Option::is_none"))] + pub idle_timeout_ms: Option, + + /// Disables TLS certificate verification for `rtsps://` URLs. + /// + /// Most cameras present self-signed certificates, which fail + /// verification against the system roots. Enabling this accepts any + /// certificate: the connection is still encrypted, but not + /// authenticated — a network attacker could impersonate the camera. + #[cfg_attr(feature = "serde", serde(default, skip_serializing_if = "std::ops::Not::not"))] + pub accept_invalid_tls_certs: bool, +} + +/// Protocol phase a timeout occurred in. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +pub enum RtspPhase { + /// Establishing the TCP connection. + Connect, + /// Waiting for the DESCRIBE response. + Describe, + /// Waiting for the SETUP response. + Setup, + /// Waiting for the PLAY response. + Play, + /// Waiting for interleaved stream data. + Stream, +} + +impl fmt::Display for RtspPhase { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Connect => "the TCP connection", + Self::Describe => "the DESCRIBE response", + Self::Setup => "the SETUP response", + Self::Play => "the PLAY response", + Self::Stream => "stream data", + }) + } +} + +/// Error returned by RTSP encoded video sources. +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum RtspVideoSourceError { + /// I/O on the RTSP connection failed. + #[error("RTSP I/O failed: {0}")] + Io(#[from] io::Error), + /// A protocol phase exceeded its timeout. + #[error("RTSP timed out waiting for {phase}")] + Timeout { + /// Protocol phase the client was waiting on. + phase: RtspPhase, + }, + /// The RTSP URL was invalid or unsupported. + #[error("invalid RTSP URL: {0}")] + InvalidUrl(&'static str), + /// The server returned a non-success status. + #[error("RTSP request failed with status {code} {reason}")] + RtspStatus { + /// RTSP status code. + code: u16, + /// RTSP status reason. + reason: String, + }, + /// A response was malformed. + #[error("invalid RTSP response: {0}")] + InvalidResponse(&'static str), + /// A response was missing a required header. + #[error("RTSP response missing {0} header")] + MissingHeader(&'static str), + /// The server requires authentication but no credentials were supplied. + #[error("RTSP authentication required but no credentials were supplied")] + MissingCredentials, + /// The authentication challenge was malformed or unsupported, or the + /// response to it could not be built. + #[error("RTSP authentication failed: {0}")] + Auth(String), + /// The SDP was missing a supported video track. + #[error("RTSP SDP does not contain a supported video track")] + MissingVideoTrack, + /// The SDP did not offer the requested codec on any video track. + #[error("RTSP SDP codec mismatch: expected {expected:?}, offered {offered:?}")] + CodecMismatch { + /// Codec required by the configuration. + expected: EncodedVideoCodec, + /// Supported codecs the SDP video tracks offered instead. + offered: Vec, + }, + /// The SDP body was malformed or not valid UTF-8. + #[error("invalid RTSP SDP")] + InvalidSdp, + /// Interleaved framing was malformed or a non-interleaved byte arrived. + #[error("unexpected RTSP interleaved data")] + UnexpectedData, + /// The URL requires TLS but the `source-rtsp-tls` feature is disabled. + #[error("rtsps:// requires the `source-rtsp-tls` feature, which is not enabled")] + TlsNotSupported, + /// TLS configuration or the TLS handshake failed. + #[error("RTSP TLS failed: {0}")] + Tls(String), + /// The stream produced no keyframe during resolution discovery. + #[error( + "stream produced no keyframe during resolution discovery; declare `resolution` in the \ + configuration to skip discovery" + )] + DiscoveryTimeout, + /// The first keyframe did not carry parseable dimensions. + #[error( + "could not determine the stream resolution from its first keyframe; declare \ + `resolution` in the configuration" + )] + DiscoveryFailed, + /// The stream ended before resolution discovery completed. + #[error("stream ended before resolution discovery completed")] + EndedDuringDiscovery, + /// The stream's resolution does not match the established one. + #[error("stream produces {actual}, but the source was configured for {configured}")] + ResolutionMismatch { + /// Resolution the source was configured for or discovered. + configured: VideoResolution, + /// Resolution the stream's keyframe declares. + actual: VideoResolution, + }, + /// RTP depacketization failed. + #[error("invalid RTP stream: {0}")] + Rtp(Box), +} + +impl From for RtspVideoSourceError { + fn from(err: rtp::RtpDepacketizerError) -> Self { + Self::Rtp(Box::new(err)) + } +} + +/// Progress from one bounded poll of the stream. +enum Poll { + AccessUnit(OwnedEncodedAccessUnit), + TimedOut, + EndOfStream, +} + +/// Encoded source that plays a video stream from an RTSP server. +pub struct RtspVideoSource { + client: RtspClient, + assembler: RtpAccessUnitAssembler, + session_id: String, + aggregate_control_url: String, + rtp_channel: u8, + keepalive_interval: Duration, + keepalive_due: Instant, + idle_timeout: Duration, + codec: EncodedVideoCodec, + resolution: VideoResolution, + resolution_verified: bool, + // Access unit pulled during resolution discovery, handed out first. + pending: Option, + eof: bool, +} + +impl fmt::Debug for RtspVideoSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RtspVideoSource") + .field("codec", &self.codec) + .field("resolution", &self.resolution) + .field("session_id", &self.session_id) + .finish_non_exhaustive() + } +} + +impl RtspVideoSource { + /// Creates the source. Connection, handshake, and resolution discovery + /// run on the tokio blocking pool. + /// + /// Requires a running tokio runtime. Use + /// [`RtspVideoSource::new_blocking`] outside of async contexts. + pub async fn new(config: RtspVideoSourceConfig) -> Result { + match tokio::task::spawn_blocking(move || Self::new_blocking(config)).await { + Ok(result) => result, + Err(err) if err.is_panic() => std::panic::resume_unwind(err.into_panic()), + Err(err) => Err(SourceError::new(err)), + } + } + + /// Connects to the RTSP server and starts playback. + /// + /// The handshake (DESCRIBE, SETUP, PLAY over TCP-interleaved transport) + /// is bounded by the connect timeout. Construction fails on an invalid + /// URL, a missing or codec-mismatched video track, an authentication + /// failure, or any non-success response. + /// + /// When the configuration declares no resolution and the SDP none + /// either, this blocks until the stream's first keyframe arrives + /// (bounded by a timeout) to read the dimensions from it. + pub fn new_blocking(config: RtspVideoSourceConfig) -> Result { + Self::connect(config).map_err(SourceError::new) + } + + fn connect(config: RtspVideoSourceConfig) -> Result { + let url = RtspUrl::parse(&config.url)?; + #[cfg(not(feature = "source-rtsp-tls"))] + if url.tls { + return Err(RtspVideoSourceError::TlsNotSupported); + } + if url.tls && config.accept_invalid_tls_certs { + log::warn!( + "TLS certificate verification is disabled for {}; \ + the connection is encrypted but the server is not authenticated", + url.request_uri, + ); + } + let credentials = merge_credentials(&config, &url); + let connect_timeout = duration_ms(config.connect_timeout_ms, DEFAULT_CONNECT_TIMEOUT); + let idle_timeout = duration_ms(config.idle_timeout_ms, DEFAULT_IDLE_TIMEOUT); + let handshake_deadline = Instant::now() + connect_timeout; + + let mut client = RtspClient::connect( + &url, + credentials, + config.accept_invalid_tls_certs, + handshake_deadline, + )?; + + let describe = client.request( + "DESCRIBE", + &url.request_uri, + &[("Accept", "application/sdp")], + handshake_deadline, + RtspPhase::Describe, + )?; + // Relative control URLs resolve against the Content-Base per + // RFC 2326 appendix C.1.1, falling back to the request URL. + let base_url = describe + .header("content-base") + .or_else(|| describe.header("content-location")) + .unwrap_or(&url.request_uri); + let session = sdp::parse_sdp_session(base_url, &describe.body, config.codec)?; + + let setup = client.request( + "SETUP", + &session.video.control_url, + &[("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1")], + handshake_deadline, + RtspPhase::Setup, + )?; + let session_header = + setup.header("session").ok_or(RtspVideoSourceError::MissingHeader("Session"))?; + let session_id = parse_session_id(session_header)?; + let session_timeout_secs = + parse_session_timeout_secs(session_header).unwrap_or(DEFAULT_SESSION_TIMEOUT_SECS); + let rtp_channel = parse_interleaved_channel(setup.header("transport"))?; + + client.request( + "PLAY", + &session.aggregate_control_url, + &[("Session", session_id.as_str()), ("Range", "npt=0.000-")], + handshake_deadline, + RtspPhase::Play, + )?; + + // Resolution declared in the configuration wins; otherwise use the + // SDP's hints (`a=framesize`, an out-of-band SPS), which spare + // waiting for media; otherwise discover from the first keyframe. + let codec = session.video.codec; + let sdp_resolution = config.resolution.or(session.video.framesize).or_else(|| { + session + .video + .parameter_sets + .sps + .iter() + .find_map(|sps| dimensions::sps_resolution(codec, sps)) + }); + + let assembler = RtpAccessUnitAssembler::new( + codec, + session.video.payload_type, + session.video.clock_rate, + session.video.parameter_sets.clone(), + sdp_resolution.unwrap_or_default(), + )?; + + let keepalive_interval = Duration::from_secs((session_timeout_secs / 2).max(1)); + let mut source = Self { + client, + assembler, + session_id, + aggregate_control_url: session.aggregate_control_url, + rtp_channel, + keepalive_interval, + keepalive_due: Instant::now() + keepalive_interval, + idle_timeout, + codec, + resolution: sdp_resolution.unwrap_or_default(), + resolution_verified: false, + pending: None, + eof: false, + }; + + // From here on the session is live, so any failure path runs the + // source's TEARDOWN through `Drop`. + if sdp_resolution.is_none() { + source.discover_resolution()?; + } + + log::info!( + "RTSP stream ready: {:?} {} ({} resolution)", + source.codec, + source.resolution, + if config.resolution.is_some() { "declared" } else { "discovered" }, + ); + Ok(source) + } + + /// Blocks until the stream's first keyframe reveals the resolution; the + /// keyframe is kept so it is not lost to discovery. + fn discover_resolution(&mut self) -> Result<(), RtspVideoSourceError> { + let deadline = Instant::now() + DISCOVERY_TIMEOUT; + loop { + match self.poll_access_unit()? { + Poll::AccessUnit(mut access_unit) => { + if access_unit.frame_type != EncodedFrameType::Key { + // Undecodable without the keyframe; the pump would + // drop these pre-roll deltas anyway. + continue; + } + let resolution = + dimensions::access_unit_resolution(self.codec, &access_unit.payload) + .ok_or(RtspVideoSourceError::DiscoveryFailed)?; + access_unit.resolution = resolution; + self.resolution = resolution; + self.assembler.set_resolution(resolution); + // The dimensions came from this keyframe itself. + self.resolution_verified = true; + self.pending = Some(access_unit); + return Ok(()); + } + Poll::TimedOut => { + if Instant::now() >= deadline { + return Err(RtspVideoSourceError::DiscoveryTimeout); + } + } + Poll::EndOfStream => return Err(RtspVideoSourceError::EndedDuringDiscovery), + } + } + } + + /// Advances the stream by at most one bounded read, running the + /// keepalive and idle-limit bookkeeping. + fn poll_access_unit(&mut self) -> Result { + loop { + if let Some(access_unit) = self.assembler.pop_ready() { + return Ok(Poll::AccessUnit(access_unit)); + } + self.maybe_send_keepalive()?; + + match self.client.poll_unit()? { + InterleavedPoll::Frame { channel, payload } if channel == self.rtp_channel => { + self.assembler.push(&payload)?; + } + // Other channels carry RTCP and non-selected media. + InterleavedPoll::Frame { .. } => {} + InterleavedPoll::Response(response) => { + // In-band responses answer keepalives; a failure (for + // example 454 Session Not Found) means the session died. + if !response.is_success() { + return Err(RtspVideoSourceError::RtspStatus { + code: response.status_code, + reason: response.reason, + }); + } + } + InterleavedPoll::TimedOut => { + if self.client.idle_for() >= self.idle_timeout { + return Err(RtspVideoSourceError::Timeout { phase: RtspPhase::Stream }); + } + return Ok(Poll::TimedOut); + } + InterleavedPoll::EndOfStream => return Ok(Poll::EndOfStream), + } + } + } + + /// Sends an OPTIONS keepalive when one is due, so the server keeps the + /// session alive across stream silence. The reply arrives in-band. + fn maybe_send_keepalive(&mut self) -> Result<(), RtspVideoSourceError> { + if Instant::now() < self.keepalive_due { + return Ok(()); + } + self.client.write_request( + "OPTIONS", + &self.aggregate_control_url, + &[("Session", self.session_id.as_str())], + )?; + self.keepalive_due = Instant::now() + self.keepalive_interval; + Ok(()) + } + + /// Verifies the established resolution against the first keyframe that + /// carries parseable dimensions. + fn verify_resolution( + &mut self, + access_unit: &OwnedEncodedAccessUnit, + ) -> Result<(), RtspVideoSourceError> { + if self.resolution_verified || access_unit.frame_type != EncodedFrameType::Key { + return Ok(()); + } + self.resolution_verified = true; + let Some(actual) = dimensions::access_unit_resolution(self.codec, &access_unit.payload) + else { + log::debug!("RTSP keyframe carries no parseable dimensions; skipping verification"); + return Ok(()); + }; + if actual != self.resolution { + return Err(RtspVideoSourceError::ResolutionMismatch { + configured: self.resolution, + actual, + }); + } + Ok(()) + } +} + +impl Drop for RtspVideoSource { + fn drop(&mut self) { + // Best-effort TEARDOWN so servers with session limits release this + // session immediately instead of waiting out its timeout. The reply + // is intentionally not awaited. + let uri = self.aggregate_control_url.clone(); + let session_id = self.session_id.clone(); + let _ = self.client.write_request("TEARDOWN", &uri, &[("Session", session_id.as_str())]); + } +} + +impl EncodedVideoSource for RtspVideoSource { + fn resolution(&self) -> VideoResolution { + self.resolution + } + + fn codec(&self) -> EncodedVideoCodec { + self.codec + } + + fn next_access_unit( + &mut self, + stop: &PumpStop, + ) -> Result, SourceError> { + if self.eof { + return Ok(None); + } + if let Some(pending) = self.pending.take() { + return Ok(Some(pending)); + } + + // Every read is bounded by the client's socket read timeout, so the + // stop token, the keepalive timer, and the idle limit are all + // observed within roughly 100ms even while the stream is silent. + loop { + if stop.is_stopped() { + return Ok(None); + } + match self.poll_access_unit().map_err(SourceError::new)? { + Poll::AccessUnit(access_unit) => { + self.verify_resolution(&access_unit).map_err(SourceError::new)?; + return Ok(Some(access_unit)); + } + Poll::TimedOut => {} + Poll::EndOfStream => { + self.eof = true; + return Ok(None); + } + } + } + } + + // TODO: Implement `request_keyframe` by sending a best-effort RTCP PLI + // on the interleaved RTCP channel (`rtp_channel + 1`), so late + // subscribers get a keyframe immediately instead of waiting out the + // producer's keyframe interval. +} + +/// Applies the config's per-field credential overrides to the URL userinfo. +fn merge_credentials( + config: &RtspVideoSourceConfig, + url: &RtspUrl, +) -> Option { + let (url_username, url_password) = match &url.credentials { + Some(credentials) => { + (Some(credentials.username.clone()), Some(credentials.password.clone())) + } + None => (None, None), + }; + let username = config.username.clone().or(url_username)?; + let password = config.password.clone().or(url_password).unwrap_or_default(); + Some(RtspCredentials { username, password }) +} + +fn duration_ms(ms: Option, default: Duration) -> Duration { + ms.map(|ms| Duration::from_millis(ms.into())).unwrap_or(default) +} + +/// Escapes text that may quote server-provided bytes before it enters an +/// error message or a log line. +fn sanitized(text: impl AsRef) -> String { + text.as_ref().escape_debug().to_string() +} + +#[cfg(test)] +mod tests { + use std::{ + io::{Read, Write}, + net::{TcpListener, TcpStream}, + thread, + }; + + use super::*; + + const SDP_H264: &str = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + const SDP_VP8: &str = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 VP8/90000\r\n"; + + fn config(url: String) -> RtspVideoSourceConfig { + RtspVideoSourceConfig { + url, + username: None, + password: None, + codec: None, + resolution: Some(VideoResolution::new(640, 480)), + connect_timeout_ms: Some(2_000), + idle_timeout_ms: None, + accept_invalid_tls_certs: false, + } + } + + #[cfg(not(feature = "source-rtsp-tls"))] + #[test] + fn rejects_rtsps_url_without_tls_feature() { + let err = RtspVideoSource::new_blocking(config("rtsps://127.0.0.1:322/camera".to_owned())) + .unwrap_err(); + assert!(err.to_string().contains("source-rtsp-tls"), "unexpected error: {err}"); + } + + fn rtp_packet(sequence_number: u16, timestamp: u32, marker: bool, payload: &[u8]) -> Vec { + let mut packet = Vec::with_capacity(12 + payload.len()); + packet.push(0x80); + packet.push(if marker { 0x80 | 96 } else { 96 }); + packet.extend_from_slice(&sequence_number.to_be_bytes()); + packet.extend_from_slice(×tamp.to_be_bytes()); + packet.extend_from_slice(&0x1122_3344_u32.to_be_bytes()); + packet.extend_from_slice(payload); + packet + } + + fn interleaved(channel: u8, payload: &[u8]) -> Vec { + let mut frame = Vec::with_capacity(4 + payload.len()); + frame.push(b'$'); + frame.push(channel); + frame.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + frame.extend_from_slice(payload); + frame + } + + fn read_request(stream: &mut impl Read) -> String { + let mut request = Vec::new(); + let mut byte = [0u8; 1]; + loop { + stream.read_exact(&mut byte).unwrap(); + request.push(byte[0]); + if request.ends_with(b"\r\n\r\n") { + break; + } + } + String::from_utf8(request).unwrap() + } + + fn write_response(stream: &mut impl Write, cseq: u32, headers: &[(&str, &str)], body: &[u8]) { + write_status_response(stream, cseq, headers, body, 200, "OK"); + } + + fn write_status_response( + stream: &mut impl Write, + cseq: u32, + headers: &[(&str, &str)], + body: &[u8], + status_code: u16, + reason: &str, + ) { + write!(stream, "RTSP/1.0 {status_code} {reason}\r\nCSeq: {cseq}\r\n").unwrap(); + for (name, value) in headers { + write!(stream, "{name}: {value}\r\n").unwrap(); + } + write!(stream, "\r\n").unwrap(); + if !body.is_empty() { + stream.write_all(body).unwrap(); + } + stream.flush().unwrap(); + } + + /// Answers DESCRIBE/SETUP/PLAY on `stream` and returns once the session + /// is playing on interleaved channels 0-1. + fn serve_handshake(stream: &mut TcpStream, sdp: &str) { + let describe = read_request(stream); + assert!(describe.starts_with("DESCRIBE rtsp://")); + write_response( + stream, + 1, + &[("Content-Type", "application/sdp"), ("Content-Length", &sdp.len().to_string())], + sdp.as_bytes(), + ); + + let setup = read_request(stream); + assert!(setup.starts_with("SETUP rtsp://")); + assert!(setup.contains("Transport: RTP/AVP/TCP;unicast;interleaved=0-1")); + write_response( + stream, + 2, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1"), + ], + &[], + ); + + let play = read_request(stream); + assert!(play.starts_with("PLAY rtsp://")); + assert!(play.contains("Session: abc123")); + write_response(stream, 3, &[], &[]); + } + + #[test] + fn connects_and_reads_rtsp_access_unit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let describe = read_request(&mut stream); + assert!(describe.starts_with("DESCRIBE rtsp://")); + write_response( + &mut stream, + 1, + &[ + ("Content-Type", "application/sdp"), + ("Content-Length", &SDP_H264.len().to_string()), + ], + SDP_H264.as_bytes(), + ); + + let setup = read_request(&mut stream); + assert!(setup.starts_with("SETUP rtsp://")); + write_response( + &mut stream, + 2, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=2-3"), + ], + &[], + ); + + let play = read_request(&mut stream); + assert!(play.starts_with("PLAY rtsp://")); + assert!(play.contains("Session: abc123")); + assert!(play.contains("Range: npt=0.000-")); + write_response(&mut stream, 3, &[], &[]); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(2, &packet)).unwrap(); + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.codec(), EncodedVideoCodec::H264); + assert_eq!(source.resolution(), VideoResolution::new(640, 480)); + assert_eq!(source.session_id, "abc123"); + assert_eq!(source.rtp_channel, 2); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn connects_with_rtsp_digest_auth() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let first_describe = read_request(&mut stream); + assert!(first_describe.starts_with(&format!("DESCRIBE rtsp://{addr}/camera"))); + assert!(!first_describe.contains("Authorization:")); + write_status_response( + &mut stream, + 1, + &[("WWW-Authenticate", "Digest realm=\"camera\", nonce=\"abcdef\", qop=\"auth\"")], + &[], + 401, + "Unauthorized", + ); + + let second_describe = read_request(&mut stream); + assert!(!second_describe.contains("admin:secret@")); + assert!(second_describe.contains("Authorization: Digest username=\"admin\"")); + assert!(second_describe.contains(&format!("uri=\"rtsp://{addr}/camera\""))); + assert!(second_describe.contains("qop=auth")); + write_response( + &mut stream, + 2, + &[ + ("Content-Type", "application/sdp"), + ("Content-Length", &SDP_H264.len().to_string()), + ], + SDP_H264.as_bytes(), + ); + + let setup = read_request(&mut stream); + assert!(setup.contains("Authorization: Digest username=\"admin\"")); + write_response( + &mut stream, + 3, + &[ + ("Session", "abc123;timeout=60"), + ("Transport", "RTP/AVP/TCP;unicast;interleaved=0-1"), + ], + &[], + ); + + let play = read_request(&mut stream); + assert!(play.contains("Authorization: Digest username=\"admin\"")); + write_response(&mut stream, 4, &[], &[]); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(0, &packet)).unwrap(); + }); + + let mut source = RtspVideoSource::new_blocking(config(format!( + "rtsp://admin:secret@{addr}/camera" + ))) + .unwrap(); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn discovers_resolution_from_first_keyframe() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_VP8); + + // VP8 keyframe with a 10-byte uncompressed header for 640x480. + let vp8_keyframe = [ + 0x10, // payload descriptor: start of partition 0 + 0x00, 0x00, 0x00, // frame tag: keyframe + 0x9d, 0x01, 0x2a, // start code + 0x80, 0x02, // width 640 + 0xe0, 0x01, // height 480 + ]; + let packet = rtp_packet(10, 12_000, true, &vp8_keyframe); + stream.write_all(&interleaved(0, &packet)).unwrap(); + read_request(&mut stream) // TEARDOWN + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: None, + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.resolution(), VideoResolution::new(640, 480)); + + // The discovery keyframe is handed out first, stamped with the + // discovered resolution. + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.resolution, VideoResolution::new(640, 480)); + drop(source); + + let teardown = server.join().unwrap(); + assert!(teardown.starts_with("TEARDOWN rtsp://")); + assert!(teardown.contains("Session: abc123")); + } + + #[test] + fn uses_sdp_framesize_without_waiting_for_media() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=0\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=framesize:96 1280-720\r\n"; + serve_handshake(&mut stream, sdp); + // Send no media: construction must not wait for any. + read_request(&mut stream) // TEARDOWN + }); + + let source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: None, + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + assert_eq!(source.resolution(), VideoResolution::new(1280, 720)); + drop(source); + server.join().unwrap(); + } + + #[test] + fn sends_keepalive_during_stream_silence() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + + // Send no interleaved data; the keepalive must arrive during the + // silence. Only then reply and send the first video frame. + let keepalive = read_request(&mut stream); + write_response(&mut stream, 4, &[], &[]); + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + stream.write_all(&interleaved(0, &packet)).unwrap(); + keepalive + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + source.keepalive_due = Instant::now() + Duration::from_millis(250); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + + let keepalive = server.join().unwrap(); + assert!(keepalive.starts_with("OPTIONS rtsp://")); + assert!(keepalive.contains("Session: abc123")); + } + + #[test] + fn failed_keepalive_reply_surfaces_as_error() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // The session died server-side: answer in-band with 454. + write_status_response(&mut stream, 4, &[], &[], 454, "Session Not Found"); + thread::sleep(Duration::from_millis(200)); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("454")); + drop(source); + server.join().unwrap(); + } + + #[test] + fn recovers_interleaved_framing_across_read_timeouts() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + let frame = interleaved(0, &packet); + // Split inside the 4-byte interleaved header and pause long + // enough for several client read timeouts in between. + let (head, tail) = frame.split_at(2); + stream.write_all(head).unwrap(); + stream.flush().unwrap(); + thread::sleep(Duration::from_millis(350)); + stream.write_all(tail).unwrap(); + stream.flush().unwrap(); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let access_unit = source.next_access_unit(&stop).unwrap().unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + drop(source); + server.join().unwrap(); + } + + #[test] + fn stream_silence_times_out_after_idle_limit() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // Stay silent past the client's idle limit. + thread::sleep(Duration::from_millis(700)); + }); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + idle_timeout_ms: Some(300), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap(); + + let stop = PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("stream data"), "unexpected error: {err}"); + drop(source); + server.join().unwrap(); + } + + #[test] + fn stop_token_is_observed_during_stream_silence() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + serve_handshake(&mut stream, SDP_H264); + // Stay silent; the client must return on its stop token alone. + thread::sleep(Duration::from_millis(700)); + }); + + let mut source = + RtspVideoSource::new_blocking(config(format!("rtsp://{addr}/camera"))).unwrap(); + + let stop = PumpStop::new(); + let stop_signal = stop.clone(); + let stopper = thread::spawn(move || { + thread::sleep(Duration::from_millis(150)); + stop_signal.stop(); + }); + + let started = Instant::now(); + let result = source.next_access_unit(&stop).unwrap(); + assert!(result.is_none()); + assert!( + started.elapsed() < Duration::from_millis(600), + "stop took {:?}", + started.elapsed() + ); + stopper.join().unwrap(); + drop(source); + server.join().unwrap(); + } + + #[test] + fn handshake_read_timeout_is_hard_error() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let server = thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let _describe = read_request(&mut stream); + // Never respond; hold the connection open past the deadline. + thread::sleep(Duration::from_millis(600)); + }); + + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + connect_timeout_ms: Some(300), + ..config(format!("rtsp://{addr}/camera")) + }) + .unwrap_err(); + + assert!( + err.to_string().contains("DESCRIBE"), + "expected DESCRIBE timeout, got: {err}" + ); + server.join().unwrap(); + } + + #[test] + fn explicit_credentials_override_url_userinfo() { + let url = RtspUrl::parse("rtsp://old:stale@camera.example/live").unwrap(); + let overriding = RtspVideoSourceConfig { + username: Some("new".to_owned()), + password: None, + ..config("rtsp://camera.example/live".to_owned()) + }; + + let credentials = merge_credentials(&overriding, &url).unwrap(); + assert_eq!(credentials.username, "new"); + // The URL password fills the unset field. + assert_eq!(credentials.password, "stale"); + + let no_credentials = merge_credentials( + &RtspVideoSourceConfig { username: None, password: None, ..overriding }, + &RtspUrl::parse("rtsp://camera.example/live").unwrap(), + ); + assert!(no_credentials.is_none()); + } +} diff --git a/livekit-capture/src/sources/rtsp/rtp/av1.rs b/livekit-capture/src/sources/rtsp/rtp/av1.rs new file mode 100644 index 000000000..3f53ae4a9 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/av1.rs @@ -0,0 +1,312 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! AV1 RTP payload handling (aomediacodec RTP payload specification). +//! +//! RTP carries OBU elements without size fields; assembly re-frames them as +//! size-prefixed OBUs so the access unit is a low-overhead AV1 bitstream. + +use super::{Av1FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::{ + encoded::EncodedFrameType, + sources::rtsp::bits::{write_leb128, BitReader, ByteReader}, +}; + +impl RtpAccessUnitAssembler { + pub(super) fn push_av1_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_av1_payload_descriptor(packet.payload)?; + if descriptor.elements.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let last_index = descriptor.elements.len() - 1; + for (index, element) in descriptor.elements.iter().enumerate() { + if element.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let obu = if index == 0 && descriptor.starts_fragment { + let Some(fragment) = self + .av1_fragment + .take() + .filter(|fragment| fragment.rtp_timestamp == packet.timestamp) + else { + // A continuation without its start means the preceding packets were lost. + self.discard_in_progress(); + return Ok(()); + }; + let mut obu = fragment.obu; + obu.extend_from_slice(element); + obu + } else { + if index == 0 && self.av1_fragment.is_some() { + return Err(RtpDepacketizerError::InvalidFragment); + } + element.to_vec() + }; + + if index == last_index && descriptor.ends_fragment { + self.av1_fragment = Some(Av1FragmentState { rtp_timestamp: packet.timestamp, obu }); + return Ok(()); + } + + let mut obu = av1_obu_from_rtp_element(&obu)?; + let frame = self.current_frame_mut(packet.timestamp)?; + if let Some(reduced_still_picture_header) = av1_reduced_still_picture_header(&obu)? { + frame.av1_reduced_still_picture_header = Some(reduced_still_picture_header); + } + if frame.frame_type.is_none() { + frame.frame_type = av1_frame_type(&obu, frame.av1_reduced_still_picture_header)?; + } + frame.payload.append(&mut obu); + } + + Ok(()) + } +} + +#[derive(Debug, Clone)] +struct Av1PayloadDescriptor<'a> { + starts_fragment: bool, + ends_fragment: bool, + elements: Vec<&'a [u8]>, +} + +fn parse_av1_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let header = reader.get_u8().ok_or_else(malformed)?; + let starts_fragment = header & 0x80 != 0; + let ends_fragment = header & 0x40 != 0; + let element_count = usize::from((header >> 4) & 0x03); + + let mut elements = Vec::new(); + if element_count == 0 { + while !reader.is_empty() { + let len = reader.get_leb128().ok_or_else(malformed)?; + elements.push(reader.take(len).ok_or_else(malformed)?); + } + } else { + for index in 0..element_count { + let element = if index + 1 == element_count { + reader.take_rest() + } else { + let len = reader.get_leb128().ok_or_else(malformed)?; + reader.take(len).ok_or_else(malformed)? + }; + elements.push(element); + } + } + + Ok(Av1PayloadDescriptor { starts_fragment, ends_fragment, elements }) +} + +/// Converts an RTP OBU element into a size-prefixed OBU. +fn av1_obu_from_rtp_element(element: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let Some(&header) = element.first() else { + return Err(malformed()); + }; + if header & 0x80 != 0 { + return Err(malformed()); + } + let header_len = if header & 0x04 != 0 { 2 } else { 1 }; + + if header & 0x02 != 0 { + // Already size-prefixed; validate the size against the element. + let mut reader = ByteReader::new(element); + reader.skip(header_len).ok_or_else(malformed)?; + let payload_size = reader.get_leb128().ok_or_else(malformed)?; + if payload_size != reader.take_rest().len() { + return Err(malformed()); + } + return Ok(element.to_vec()); + } + + let payload = element.get(header_len..).ok_or_else(malformed)?; + let mut obu = Vec::with_capacity(element.len() + 8); + obu.push(header | 0x02); + if header & 0x04 != 0 { + obu.push(element[1]); + } + write_leb128(payload.len(), &mut obu); + obu.extend_from_slice(payload); + Ok(obu) +} + +/// Reads `reduced_still_picture_header` from a sequence header OBU. +fn av1_reduced_still_picture_header(obu: &[u8]) -> Result, RtpDepacketizerError> { + let Some((obu_type, payload)) = av1_obu_parts(obu)? else { + return Ok(None); + }; + if obu_type != 1 { + return Ok(None); + } + + let mut reader = BitReader::new(payload); + reader.read_bits(3).ok_or(RtpDepacketizerError::UnsupportedPayload)?; // seq_profile + reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)?; // still_picture + Ok(Some(reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)? != 0)) +} + +/// Classifies a frame or frame-header OBU, when `obu` is one. +fn av1_frame_type( + obu: &[u8], + reduced_still_picture_header: Option, +) -> Result, RtpDepacketizerError> { + let Some((obu_type, payload)) = av1_obu_parts(obu)? else { + return Ok(None); + }; + if !matches!(obu_type, 3 | 6) { + return Ok(None); + } + + if reduced_still_picture_header.unwrap_or(false) { + return Ok(Some(EncodedFrameType::Key)); + } + + let mut reader = BitReader::new(payload); + let show_existing_frame = reader.read_bit().ok_or(RtpDepacketizerError::UnsupportedPayload)?; + if show_existing_frame != 0 { + return Ok(Some(EncodedFrameType::Delta)); + } + + let frame_type = reader.read_bits(2).ok_or(RtpDepacketizerError::UnsupportedPayload)?; + Ok(Some(if frame_type == 0 { EncodedFrameType::Key } else { EncodedFrameType::Delta })) +} + +/// Splits an OBU into its type and payload bytes. +fn av1_obu_parts(obu: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let Some(&header) = obu.first() else { + return Ok(None); + }; + if header & 0x80 != 0 { + return Err(malformed()); + } + + let obu_type = (header & 0x78) >> 3; + let has_extension = header & 0x04 != 0; + let has_size = header & 0x02 != 0; + let mut reader = ByteReader::new(obu); + reader.skip(if has_extension { 2 } else { 1 }).ok_or_else(malformed)?; + + if !has_size { + return Ok(Some((obu_type, reader.take_rest()))); + } + + let payload_size = reader.get_leb128().ok_or_else(malformed)?; + let payload = reader.take(payload_size).ok_or_else(malformed)?; + Ok(Some((obu_type, payload))) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::encoded::EncodedVideoCodec; + + fn av1_sequence_and_frame_rtp_payload(frame_header: u8) -> [u8; 6] { + [ + 0x28, // W=2, N=1. + 0x02, // First OBU element length. + 0x08, // Sequence header OBU without the size field. + 0x00, // profile=0, still_picture=false, reduced_still_picture_header=false. + 0x30, // Frame OBU without the size field. + frame_header, + ] + } + + #[test] + fn assembles_av1_temporal_unit() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::AV1); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + } + + #[test] + fn av1_sequence_header_before_inter_frame_is_delta() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &av1_sequence_and_frame_rtp_payload(0x38)); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x38]); + } + + #[test] + fn assembles_fragmented_av1_obu() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let start = rtp_packet(10, 12_000, false, &[0x50, 0x30, 0x38]); + let end = rtp_packet(11, 12_000, true, &[0x90, 2, 3]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x32, 0x03, 0x38, 2, 3]); + } + + #[test] + fn assembles_av1_obu_payload_with_size_field() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + let packet = rtp_packet(10, 12_000, true, &[0x10, 0x30, 0x38, 2, 3]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x32, 0x03, 0x38, 2, 3]); + } + + #[test] + fn marker_with_open_av1_fragment_drops_frame() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + // Y is set, so the OBU fragment is unterminated when the marker closes it. + let truncated = rtp_packet(10, 12_000, true, &[0x50, 0x30, 1]); + let key = rtp_packet(11, 15_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + assert!(push_one(&mut assembler, &truncated).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn drops_av1_fragment_continuation_without_start() { + let mut assembler = assembler(EncodedVideoCodec::AV1); + // Z is set: this continues an OBU whose start was never received. + let continuation = rtp_packet(10, 12_000, true, &[0x90, 2, 3]); + let key = rtp_packet(11, 15_000, true, &av1_sequence_and_frame_rtp_payload(0x10)); + + assert!(push_one(&mut assembler, &continuation).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x0a, 0x01, 0x00, 0x32, 0x01, 0x10]); + } +} diff --git a/livekit-capture/src/sources/rtsp/rtp/h26x.rs b/livekit-capture/src/sources/rtsp/rtp/h26x.rs new file mode 100644 index 000000000..894b0adc9 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/h26x.rs @@ -0,0 +1,591 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! H.264 (RFC 6184) and H.265 (RFC 7798) RTP payload handling. + +use super::{FragmentState, RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::{ + encoded::{ + h26x::{ + access_unit_from_nalus, avc_nalus, h264_nal_type, h265_nal_type, H264_NAL_IDR, + H264_NAL_PPS, H264_NAL_SPS, H265_NAL_IDR_N_LP, H265_NAL_IDR_W_RADL, H265_NAL_PPS, + H265_NAL_SPS, H265_NAL_VPS, + }, + EncodedFrameType, EncodedVideoCodec, + }, + sources::rtsp::bits::ByteReader, +}; + +/// One parsed H.264 RTP payload (RFC 6184). +#[derive(Debug, PartialEq, Eq)] +enum H264Payload<'a> { + /// A complete NAL unit (types 1-23), including its header. + Nal(&'a [u8]), + /// STAP-A: aggregated complete NAL units. + Aggregation(Vec<&'a [u8]>), + /// FU-A: a fragment of one NAL unit. + Fragment(Fragment<'a>), +} + +/// One parsed H.265 RTP payload (RFC 7798). +#[derive(Debug, PartialEq, Eq)] +enum H265Payload<'a> { + /// A complete NAL unit (types 0-47), including its header. + Nal(&'a [u8]), + /// AP: aggregated complete NAL units. + Aggregation(Vec<&'a [u8]>), + /// FU: a fragment of one NAL unit. + Fragment(Fragment<'a>), +} + +/// A fragment of one NAL unit. +#[derive(Debug, PartialEq, Eq)] +struct Fragment<'a> { + /// Whether this is the first fragment of the NAL unit. + start: bool, + /// Whether this is the last fragment of the NAL unit. + end: bool, + /// The fragmented NAL unit's header, reconstructed from the + /// fragmentation headers. + nal_header: NalHeader, + /// Fragment payload bytes. + payload: &'a [u8], +} + +/// A reconstructed NAL unit header. +#[derive(Debug, PartialEq, Eq)] +enum NalHeader { + H264(u8), + H265([u8; 2]), +} + +impl NalHeader { + fn as_slice(&self) -> &[u8] { + match self { + Self::H264(header) => std::slice::from_ref(header), + Self::H265(header) => header, + } + } +} + +/// Parses one H.264 RTP payload without touching assembly state. +fn parse_h264_payload(payload: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + let indicator = reader.get_u8().ok_or_else(malformed)?; + + match indicator & 0x1f { + 1..=23 => Ok(H264Payload::Nal(payload)), + // STAP-A: 2-byte-length-prefixed NAL units, like an AVC access unit. + 24 => Ok(H264Payload::Aggregation(avc_nalus(reader.take_rest(), 2)?)), + 28 => { + let fu_header = reader.get_u8().ok_or_else(malformed)?; + let nal_type = fu_header & 0x1f; + if nal_type == 0 || nal_type > 23 { + return Err(malformed()); + } + Ok(H264Payload::Fragment(Fragment { + start: fu_header & 0x80 != 0, + end: fu_header & 0x40 != 0, + nal_header: NalHeader::H264((indicator & 0xe0) | nal_type), + payload: reader.take_rest(), + })) + } + _ => Err(malformed()), + } +} + +/// Parses one H.265 RTP payload without touching assembly state. +fn parse_h265_payload(payload: &[u8]) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + let header = [ + reader.get_u8().ok_or_else(malformed)?, + reader.get_u8().ok_or_else(malformed)?, + ]; + + match (header[0] >> 1) & 0x3f { + 0..=47 => Ok(H265Payload::Nal(payload)), + // AP: 2-byte-length-prefixed NAL units, like an AVC access unit. + 48 => Ok(H265Payload::Aggregation(avc_nalus(reader.take_rest(), 2)?)), + 49 => { + let fu_header = reader.get_u8().ok_or_else(malformed)?; + let nal_type = fu_header & 0x3f; + if nal_type > 47 { + return Err(malformed()); + } + Ok(H265Payload::Fragment(Fragment { + start: fu_header & 0x80 != 0, + end: fu_header & 0x40 != 0, + nal_header: NalHeader::H265([(header[0] & 0x81) | (nal_type << 1), header[1]]), + payload: reader.take_rest(), + })) + } + _ => Err(malformed()), + } +} + +impl RtpAccessUnitAssembler { + pub(super) fn push_h264_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + match parse_h264_payload(packet.payload)? { + H264Payload::Nal(nal_unit) => { + self.current_mut(packet.timestamp)?.nal_units.push(nal_unit.to_vec()); + } + H264Payload::Aggregation(nal_units) => { + self.push_aggregation(packet.timestamp, nal_units)?; + } + H264Payload::Fragment(fragment) => self.apply_fragment(packet.timestamp, fragment)?, + } + Ok(()) + } + + pub(super) fn push_h265_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + match parse_h265_payload(packet.payload)? { + H265Payload::Nal(nal_unit) => { + self.current_mut(packet.timestamp)?.nal_units.push(nal_unit.to_vec()); + } + H265Payload::Aggregation(nal_units) => { + self.push_aggregation(packet.timestamp, nal_units)?; + } + H265Payload::Fragment(fragment) => self.apply_fragment(packet.timestamp, fragment)?, + } + Ok(()) + } + + /// Adds the complete NAL units of an aggregation packet. + fn push_aggregation( + &mut self, + rtp_timestamp: u32, + nal_units: Vec<&[u8]>, + ) -> Result<(), RtpDepacketizerError> { + let nal_units: Vec> = nal_units.into_iter().map(<[u8]>::to_vec).collect(); + self.current_mut(rtp_timestamp)?.nal_units.extend(nal_units); + Ok(()) + } + + /// Applies one NAL fragment: starts, extends, or completes the pending + /// fragmented NAL unit. + fn apply_fragment( + &mut self, + rtp_timestamp: u32, + fragment: Fragment<'_>, + ) -> Result<(), RtpDepacketizerError> { + if fragment.start { + let nal_header = fragment.nal_header.as_slice(); + let mut nal_unit = Vec::with_capacity(nal_header.len() + fragment.payload.len()); + nal_unit.extend_from_slice(nal_header); + nal_unit.extend_from_slice(fragment.payload); + self.fragment = Some(FragmentState { rtp_timestamp, nal_unit }); + return Ok(()); + } + + let Some(state) = + self.fragment.as_mut().filter(|state| state.rtp_timestamp == rtp_timestamp) + else { + // A continuation without its start means the preceding packets were lost. + self.discard_in_progress(); + return Ok(()); + }; + state.nal_unit.extend_from_slice(fragment.payload); + + if fragment.end { + let nal_unit = + self.fragment.take().ok_or(RtpDepacketizerError::InvalidFragment)?.nal_unit; + self.current_mut(rtp_timestamp)?.nal_units.push(nal_unit); + } + Ok(()) + } + + /// Completes the pending H.26x access unit and queues it. + /// + /// Keyframe access units missing parameter sets receive the ones the SDP + /// carried out-of-band, so every published keyframe is self-contained: + /// passthrough subscribers may join mid-stream and can only initialize + /// their decoder from parameter sets inside the keyframe itself. + pub(super) fn finish_current(&mut self) -> Result<(), RtpDepacketizerError> { + // An open fragment carrying into the next unit undercounts by at + // most one NAL's fragment, which the generous cap absorbs. + self.pending_bytes = 0; + let Some(current) = self.current.take() else { + return Ok(()); + }; + if current.nal_units.is_empty() { + return Ok(()); + } + + let mut presence = NalPresence::default(); + for nal in ¤t.nal_units { + presence.record(self.codec, nal); + } + + // Prepend only the missing parameter-set kinds, in VPS, SPS, PPS + // order; in-band parameter sets pass through untouched. + let mut nal_units = Vec::with_capacity(current.nal_units.len() + 3); + if presence.idr { + if self.codec == EncodedVideoCodec::H265 && !presence.vps { + nal_units.extend(self.parameter_sets.vps.iter().map(Vec::as_slice)); + } + if !presence.sps { + nal_units.extend(self.parameter_sets.sps.iter().map(Vec::as_slice)); + } + if !presence.pps { + nal_units.extend(self.parameter_sets.pps.iter().map(Vec::as_slice)); + } + } + nal_units.extend(current.nal_units.iter().map(Vec::as_slice)); + + let access_unit = + access_unit_from_nalus(self.codec, &nal_units, current.timestamp_us, self.resolution)?; + + if presence.idr + && access_unit.frame_type != EncodedFrameType::Key + && !self.warned_missing_parameter_sets + { + self.warned_missing_parameter_sets = true; + log::warn!( + "H.265 keyframe lacks VPS/SPS/PPS and the SDP provided none; \ + the stream cannot be published until a self-contained keyframe arrives" + ); + } + + self.enqueue(access_unit); + Ok(()) + } +} + +/// Which access-unit-defining NAL kinds appear in a pending access unit. +#[derive(Debug, Clone, Copy, Default)] +struct NalPresence { + vps: bool, + sps: bool, + pps: bool, + idr: bool, +} + +impl NalPresence { + fn record(&mut self, codec: EncodedVideoCodec, nal: &[u8]) { + match codec { + EncodedVideoCodec::H264 => match h264_nal_type(nal) { + Ok(H264_NAL_IDR) => self.idr = true, + Ok(H264_NAL_SPS) => self.sps = true, + Ok(H264_NAL_PPS) => self.pps = true, + _ => {} + }, + EncodedVideoCodec::H265 => match h265_nal_type(nal) { + Ok(H265_NAL_IDR_W_RADL | H265_NAL_IDR_N_LP) => self.idr = true, + Ok(H265_NAL_VPS) => self.vps = true, + Ok(H265_NAL_SPS) => self.sps = true, + Ok(H265_NAL_PPS) => self.pps = true, + _ => {} + }, + _ => {} + } + } +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::{ + encoded::OwnedEncodedAccessUnit, primitive::VideoResolution, + sources::rtsp::rtp::H26xParameterSets, + }; + + fn assembler_with_parameter_sets( + codec: EncodedVideoCodec, + parameter_sets: H26xParameterSets, + ) -> RtpAccessUnitAssembler { + RtpAccessUnitAssembler::new( + codec, + 96, + 90_000, + parameter_sets, + VideoResolution::new(640, 480), + ) + .unwrap() + } + + fn annex_b_nals(access_unit: &OwnedEncodedAccessUnit) -> Vec<&[u8]> { + crate::encoded::h26x::annex_b_nalus(&access_unit.payload) + } + + #[test] + fn parses_h264_payloads() { + assert_eq!(parse_h264_payload(&[0x65, 1, 2]), Ok(H264Payload::Nal(&[0x65, 1, 2]))); + assert_eq!( + parse_h264_payload(&[0x18, 0, 2, 0x67, 9, 0, 1, 0x68]), + Ok(H264Payload::Aggregation(vec![&[0x67, 9][..], &[0x68][..]])) + ); + // FU-A start: the NAL header is rebuilt from the indicator's NRI + // bits and the FU header's type. + assert_eq!( + parse_h264_payload(&[0x7c, 0x85, 1, 2]), + Ok(H264Payload::Fragment(Fragment { + start: true, + end: false, + nal_header: NalHeader::H264(0x65), + payload: &[1, 2], + })) + ); + // FU-A end. + assert_eq!( + parse_h264_payload(&[0x7c, 0x45, 3]), + Ok(H264Payload::Fragment(Fragment { + start: false, + end: true, + nal_header: NalHeader::H264(0x65), + payload: &[3], + })) + ); + // Truncated, forbidden fragment types, and unsupported packet types. + assert!(parse_h264_payload(&[]).is_err()); + assert!(parse_h264_payload(&[0x7c]).is_err()); + assert!(parse_h264_payload(&[0x7c, 0x80]).is_err()); // FU of type 0 + assert!(parse_h264_payload(&[0x19, 1]).is_err()); // STAP-B + } + + #[test] + fn parses_h265_payloads() { + assert_eq!( + parse_h265_payload(&[0x26, 0x01, 1]), + Ok(H265Payload::Nal(&[0x26, 0x01, 1])) + ); + assert_eq!( + parse_h265_payload(&[0x60, 0x01, 0, 2, 0x40, 0x01]), + Ok(H265Payload::Aggregation(vec![&[0x40, 0x01][..]])) + ); + // FU start: the 2-byte NAL header keeps the layer and temporal-id + // bits and takes the type from the FU header. + assert_eq!( + parse_h265_payload(&[0x62, 0x01, 0x93, 1, 2]), + Ok(H265Payload::Fragment(Fragment { + start: true, + end: false, + nal_header: NalHeader::H265([0x26, 0x01]), + payload: &[1, 2], + })) + ); + // Truncated inputs and reserved fragment types. + assert!(parse_h265_payload(&[0x62]).is_err()); + assert!(parse_h265_payload(&[0x62, 0x01]).is_err()); + assert!(parse_h265_payload(&[0x62, 0x01, 0xb0]).is_err()); // FU of type 48 + } + + #[test] + fn assembles_h264_single_nal_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let packet = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + } + + #[test] + fn assembles_h264_stap_a() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // STAP-A carrying SPS (2 bytes) and PPS (2 bytes), then an IDR. + let stap = rtp_packet(10, 12_000, false, &[0x18, 0, 2, 0x67, 9, 0, 2, 0x68, 8]); + let idr = rtp_packet(11, 12_000, true, &[0x65, 1]); + + assert!(push_one(&mut assembler, &stap).is_none()); + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 9, 0, 0, 0, 1, 0x68, 8, 0, 0, 0, 1, 0x65, 1] + ); + } + + #[test] + fn rejects_empty_h264_aggregation() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // A STAP-A with no NAL entries is malformed. + let empty_stap = rtp_packet(10, 12_000, true, &[0x18]); + assert!(assembler.push(&empty_stap).is_err()); + } + + #[test] + fn assembles_h264_fu_a() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x7c, 0x45, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2, 3, 4]); + } + + #[test] + fn sequence_gap_recovers_h264_at_next_keyframe() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let delta = rtp_packet(12, 15_000, true, &[0x41, 1, 2]); + let key = rtp_packet(13, 18_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The gap dropped the fragment; the delta frame after it is withheld. + assert!(push_one(&mut assembler, &delta).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 1); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 3, 4]); + let stats = assembler.stats(); + assert_eq!(stats.dropped_access_units, 1); + assert!(!stats.awaiting_keyframe); + } + + #[test] + fn marker_with_open_h264_fragment_drops_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + let truncated = rtp_packet(11, 12_000, true, &[0x7c, 0x05, 3, 4]); + let key = rtp_packet(12, 15_000, true, &[0x65, 5, 6]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The marker arrived without the FU end bit: the fragment is truncated. + assert!(push_one(&mut assembler, &truncated).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 0); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 5, 6]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn drops_h264_fu_continuation_without_start() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let continuation = rtp_packet(10, 12_000, false, &[0x7c, 0x05, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &continuation).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 3, 4]); + } + + #[test] + fn assembles_h265_fragment_units() { + let mut assembler = assembler(EncodedVideoCodec::H265); + // FU (type 49) carrying an IDR_W_RADL (type 19) split in two, after + // an AP (type 48) carrying VPS, SPS, and PPS. + let parameter_sets = rtp_packet( + 9, + 12_000, + false, + &[ + 0x60, 0x01, // AP NAL header. + 0, 2, 0x40, 0x01, // VPS (type 32). + 0, 2, 0x42, 0x01, // SPS (type 33). + 0, 2, 0x44, 0x01, // PPS (type 34). + ], + ); + let start = rtp_packet(10, 12_000, false, &[0x62, 0x01, 0x93, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x62, 0x01, 0x53, 3, 4]); + + assert!(push_one(&mut assembler, ¶meter_sets).is_none()); + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + let nals = annex_b_nals(&access_unit); + assert_eq!(nals.len(), 4); + assert_eq!(nals[3], &[0x26, 0x01, 1, 2, 3, 4]); + } + + #[test] + fn injects_sdp_parameter_sets_into_h264_keyframe() { + let parameter_sets = H26xParameterSets { + vps: Vec::new(), + sps: vec![vec![0x67, 9, 8]], + pps: vec![vec![0x68, 7]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H264, parameter_sets); + let idr = rtp_packet(10, 12_000, true, &[0x65, 1, 2]); + let delta = rtp_packet(11, 15_000, true, &[0x41, 3]); + + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 9, 8, 0, 0, 0, 1, 0x68, 7, 0, 0, 0, 1, 0x65, 1, 2] + ); + + // Delta frames pass through untouched. + let access_unit = push_one(&mut assembler, &delta).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x41, 3]); + } + + #[test] + fn injects_sdp_parameter_sets_into_h265_keyframe() { + let parameter_sets = H26xParameterSets { + vps: vec![vec![0x40, 0x01, 1]], + sps: vec![vec![0x42, 0x01, 2]], + pps: vec![vec![0x44, 0x01, 3]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H265, parameter_sets); + // An IDR-only access unit classifies as a keyframe only once the SDP + // parameter sets are injected. + let idr = rtp_packet(10, 12_000, true, &[0x26, 0x01, 1, 2]); + + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + let nals = annex_b_nals(&access_unit); + assert_eq!(nals, vec![ + &[0x40, 0x01, 1][..], + &[0x42, 0x01, 2][..], + &[0x44, 0x01, 3][..], + &[0x26, 0x01, 1, 2][..], + ]); + } + + #[test] + fn does_not_duplicate_in_band_parameter_sets() { + let parameter_sets = H26xParameterSets { + vps: Vec::new(), + sps: vec![vec![0x67, 99]], + pps: vec![vec![0x68, 99]], + }; + let mut assembler = + assembler_with_parameter_sets(EncodedVideoCodec::H264, parameter_sets); + // The stream repeats its own parameter sets in-band. + let stap = rtp_packet(10, 12_000, false, &[0x18, 0, 2, 0x67, 1, 0, 2, 0x68, 2]); + let idr = rtp_packet(11, 12_000, true, &[0x65, 3]); + + assert!(push_one(&mut assembler, &stap).is_none()); + let access_unit = push_one(&mut assembler, &idr).unwrap(); + assert_eq!( + access_unit.payload.as_ref(), + &[0, 0, 0, 1, 0x67, 1, 0, 0, 0, 1, 0x68, 2, 0, 0, 0, 1, 0x65, 3] + ); + } +} diff --git a/livekit-capture/src/sources/rtsp/rtp/mod.rs b/livekit-capture/src/sources/rtsp/rtp/mod.rs new file mode 100644 index 000000000..a81cd8071 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/mod.rs @@ -0,0 +1,735 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTP depacketization into encoded access units. +//! +//! [`RtpAccessUnitAssembler`] reassembles the RTP payload formats of the +//! supported codecs and recovers from packet loss by discarding the +//! interrupted access unit and gating output on the next keyframe. + +mod av1; +mod h26x; +mod vpx; + +use std::collections::VecDeque; + +use thiserror::Error; + +use crate::{ + encoded::{ + h26x::{H26xParseError, MAX_PENDING_ACCESS_UNIT_BYTES}, + EncodedFrameType, EncodedVideoCodec, OwnedEncodedAccessUnit, + }, + primitive::VideoResolution, + sources::rtsp::bits::ByteReader, +}; + +/// Out-of-band H.26x parameter sets, decoded from SDP `fmtp` attributes. +/// +/// The assembler prepends missing parameter sets to keyframe access units so +/// every published keyframe is self-contained; see +/// [`RtpAccessUnitAssembler::finish_current`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub(super) struct H26xParameterSets { + /// H.265 video parameter sets (NAL type 32). + pub(super) vps: Vec>, + /// Sequence parameter sets (H.264 NAL type 7, H.265 NAL type 33). + pub(super) sps: Vec>, + /// Picture parameter sets (H.264 NAL type 8, H.265 NAL type 34). + pub(super) pps: Vec>, +} + +impl H26xParameterSets { + /// Returns `true` when no parameter sets were provided. + #[cfg(test)] + pub(super) fn is_empty(&self) -> bool { + self.vps.is_empty() && self.sps.is_empty() && self.pps.is_empty() + } +} + +/// Parsed RTP packet header and payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct RtpPacket<'a> { + /// RTP marker bit. + pub(super) marker: bool, + /// RTP payload type. + pub(super) payload_type: u8, + /// RTP sequence number. + pub(super) sequence_number: u16, + /// RTP timestamp. + pub(super) timestamp: u32, + /// RTP SSRC. + pub(super) ssrc: u32, + /// RTP payload bytes. + pub(super) payload: &'a [u8], +} + +impl<'a> RtpPacket<'a> { + /// Parses a single RTP packet. + pub(super) fn parse(bytes: &'a [u8]) -> Result { + let too_short = || RtpDepacketizerError::PacketTooShort; + if bytes.len() < 12 { + return Err(too_short()); + } + let mut reader = ByteReader::new(bytes); + + let flags = reader.get_u8().ok_or_else(too_short)?; + if flags >> 6 != 2 { + return Err(RtpDepacketizerError::UnsupportedVersion(flags >> 6)); + } + let has_padding = (flags & 0x20) != 0; + let has_extension = (flags & 0x10) != 0; + let csrc_count = (flags & 0x0f) as usize; + + let marker_and_type = reader.get_u8().ok_or_else(too_short)?; + let sequence_number = reader.get_u16_be().ok_or_else(too_short)?; + let timestamp = reader.get_u32_be().ok_or_else(too_short)?; + let ssrc = reader.get_u32_be().ok_or_else(too_short)?; + reader.skip(csrc_count * 4).ok_or_else(too_short)?; + + if has_extension { + reader.skip(2).ok_or_else(too_short)?; // profile-specific identifier + let extension_words = reader.get_u16_be().ok_or_else(too_short)? as usize; + reader.skip(extension_words * 4).ok_or_else(too_short)?; + } + + let mut payload = reader.take_rest(); + if has_padding { + let padding = *payload.last().ok_or_else(too_short)? as usize; + if padding == 0 || padding > payload.len() { + return Err(RtpDepacketizerError::PacketTooShort); + } + payload = &payload[..payload.len() - padding]; + } + + Ok(Self { + marker: (marker_and_type & 0x80) != 0, + payload_type: marker_and_type & 0x7f, + sequence_number, + timestamp, + ssrc, + payload, + }) + } +} + +/// Maps RTP timestamps to capture timestamps in microseconds. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct RtpTimestampMapper { + clock_rate: u32, + last_rtp_timestamp: Option, + extended_ticks: i64, + base_timestamp_us: i64, +} + +impl RtpTimestampMapper { + /// Creates an RTP timestamp mapper with a non-zero clock rate. + fn new(clock_rate: u32, base_timestamp_us: i64) -> Result { + if clock_rate == 0 { + return Err(RtpDepacketizerError::InvalidClockRate); + } + Ok(Self { clock_rate, last_rtp_timestamp: None, extended_ticks: 0, base_timestamp_us }) + } + + /// Maps an RTP timestamp to microseconds, unwrapping `u32` RTP timestamp + /// rollover so mapped timestamps stay monotonic across any number of wraps. + fn map(&mut self, rtp_timestamp: u32) -> i64 { + let last = *self.last_rtp_timestamp.get_or_insert(rtp_timestamp); + self.last_rtp_timestamp = Some(rtp_timestamp); + // Reinterpreting the wrapped u32 delta as i32 picks the nearest extended + // timestamp, which unwraps rollover while tolerating small backwards + // jumps from reordered packets. + let delta_ticks = i64::from(rtp_timestamp.wrapping_sub(last) as i32); + self.extended_ticks = self.extended_ticks.saturating_add(delta_ticks); + + let extended_us = i128::from(self.extended_ticks) * 1_000_000 / i128::from(self.clock_rate); + let extended_us = extended_us.clamp(i128::from(i64::MIN), i128::from(i64::MAX)) as i64; + self.base_timestamp_us.saturating_add(extended_us) + } +} + +/// Error returned by RTP depacketization and access-unit assembly. +#[derive(Debug, Error, PartialEq, Eq)] +pub(super) enum RtpDepacketizerError { + /// RTP packet is shorter than its declared header. + #[error("RTP packet is too short")] + PacketTooShort, + /// RTP version is not supported. + #[error("unsupported RTP version {0}")] + UnsupportedVersion(u8), + /// RTP clock rate must be non-zero. + #[error("RTP clock rate must be non-zero")] + InvalidClockRate, + /// RTP payload format is unsupported or malformed. + #[error("unsupported or malformed RTP payload")] + UnsupportedPayload, + /// RTP fragmentation state was invalid. + #[error("invalid RTP fragmentation sequence")] + InvalidFragment, + /// The payload descriptor is unsupported by the single-layer depacketizer. + #[error("unsupported RTP payload descriptor")] + UnsupportedPayloadDescriptor, + /// Assembled NAL units could not form an access unit. + #[error(transparent)] + H26x(#[from] H26xParseError), +} + +/// Reassembles the RTP packets of one video stream into encoded access units. +/// +/// Packets whose payload type or SSRC does not match the negotiated stream +/// are ignored. Packet loss is recovered internally: gaps and truncated +/// fragments drop the interrupted access unit and gate output on the next +/// keyframe instead of returning an error. +#[derive(Debug, Clone)] +pub(super) struct RtpAccessUnitAssembler { + codec: EncodedVideoCodec, + payload_type: u8, + /// SSRC latched from the first matching packet; later packets from other + /// sources on the same channel are dropped. + ssrc: Option, + resolution: VideoResolution, + parameter_sets: H26xParameterSets, + timestamp_mapper: RtpTimestampMapper, + expected_sequence_number: Option, + current: Option, + fragment: Option, + current_frame: Option, + av1_fragment: Option, + ready: VecDeque, + awaiting_keyframe: bool, + /// Payload bytes accumulated since the last completed or discarded + /// access unit, bounding what an endless unit can buffer. + pending_bytes: usize, + logged_ssrc_mismatch: bool, + warned_missing_parameter_sets: bool, + warned_oversized_pending: bool, + sequence_gaps: u64, + dropped_access_units: u64, +} + +#[derive(Debug, Clone)] +struct PartialAccessUnit { + rtp_timestamp: u32, + timestamp_us: i64, + nal_units: Vec>, +} + +#[derive(Debug, Clone)] +struct FragmentState { + rtp_timestamp: u32, + nal_unit: Vec, +} + +#[derive(Debug, Clone)] +struct PartialFrame { + rtp_timestamp: u32, + timestamp_us: i64, + payload: Vec, + frame_type: Option, + av1_reduced_still_picture_header: Option, +} + +#[derive(Debug, Clone)] +struct Av1FragmentState { + rtp_timestamp: u32, + obu: Vec, +} + +/// Packet-loss recovery counters for an [`RtpAccessUnitAssembler`]. +#[cfg(test)] +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub(super) struct RtpDepacketizerStats { + pub(super) sequence_gaps: u64, + pub(super) dropped_access_units: u64, + pub(super) awaiting_keyframe: bool, +} + +impl RtpAccessUnitAssembler { + /// Creates an RTP access-unit assembler for one negotiated video stream. + pub(super) fn new( + codec: EncodedVideoCodec, + payload_type: u8, + clock_rate: u32, + parameter_sets: H26xParameterSets, + resolution: VideoResolution, + ) -> Result { + Ok(Self { + codec, + payload_type, + ssrc: None, + resolution, + parameter_sets, + timestamp_mapper: RtpTimestampMapper::new(clock_rate, 0)?, + expected_sequence_number: None, + current: None, + fragment: None, + current_frame: None, + av1_fragment: None, + ready: VecDeque::new(), + awaiting_keyframe: false, + pending_bytes: 0, + logged_ssrc_mismatch: false, + warned_missing_parameter_sets: false, + warned_oversized_pending: false, + sequence_gaps: 0, + dropped_access_units: 0, + }) + } + + /// Sets the resolution stamped on access units assembled from now on. + pub(super) fn set_resolution(&mut self, resolution: VideoResolution) { + self.resolution = resolution; + } + + /// Returns the next completed access unit, if any. + pub(super) fn pop_ready(&mut self) -> Option { + self.ready.pop_front() + } + + /// Returns packet-loss recovery counters. + #[cfg(test)] + pub(super) fn stats(&self) -> RtpDepacketizerStats { + RtpDepacketizerStats { + sequence_gaps: self.sequence_gaps, + dropped_access_units: self.dropped_access_units, + awaiting_keyframe: self.awaiting_keyframe, + } + } + + /// Pushes one encoded RTP packet; completed access units become available + /// through [`Self::pop_ready`]. + pub(super) fn push(&mut self, bytes: &[u8]) -> Result<(), RtpDepacketizerError> { + let packet = RtpPacket::parse(bytes)?; + if packet.payload_type != self.payload_type { + return Ok(()); + } + match self.ssrc { + None => self.ssrc = Some(packet.ssrc), + Some(ssrc) if ssrc != packet.ssrc => { + if !self.logged_ssrc_mismatch { + log::debug!( + "dropping RTP packets from unexpected SSRC {:#010x}; \ + the stream is locked to SSRC {ssrc:#010x}", + packet.ssrc, + ); + self.logged_ssrc_mismatch = true; + } + return Ok(()); + } + Some(_) => {} + } + + self.check_sequence(packet.sequence_number); + self.finish_on_timestamp_change(packet.timestamp)?; + + match self.codec { + EncodedVideoCodec::H264 => self.push_h264_payload(&packet)?, + EncodedVideoCodec::H265 => self.push_h265_payload(&packet)?, + EncodedVideoCodec::VP8 => self.push_vp8_payload(&packet)?, + EncodedVideoCodec::VP9 => self.push_vp9_payload(&packet)?, + EncodedVideoCodec::AV1 => self.push_av1_payload(&packet)?, + } + + // Bound what an unfinished access unit can buffer: a stream that + // never completes one must not grow memory without limit. Counting + // raw payload bytes slightly overestimates, which only trips the + // generous cap earlier. + self.pending_bytes = self.pending_bytes.saturating_add(packet.payload.len()); + if self.pending_bytes > MAX_PENDING_ACCESS_UNIT_BYTES { + if !self.warned_oversized_pending { + self.warned_oversized_pending = true; + log::warn!( + "discarding an access unit still incomplete after \ + {MAX_PENDING_ACCESS_UNIT_BYTES} buffered bytes; \ + waiting for the next keyframe" + ); + } + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + + if packet.marker { + if self.fragment.is_some() || self.av1_fragment.is_some() { + // The marker closed the access unit before the open fragment's + // end arrived, so its tail packets were lost. + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + self.finish_pending()?; + } + Ok(()) + } + + fn check_sequence(&mut self, sequence_number: u16) { + let Some(expected) = self.expected_sequence_number.replace(sequence_number.wrapping_add(1)) + else { + return; + }; + if sequence_number == expected { + return; + } + + self.sequence_gaps += 1; + self.discard_in_progress(); + } + + /// Completes the pending access unit when a packet starts a new RTP + /// timestamp without a marker having closed the previous one. + /// + /// Some producers never set the RTP marker bit; on a contiguous sequence, + /// a timestamp change still proves the previous access unit is complete. + /// A lost marker packet is a sequence gap instead, which discards the + /// interrupted access unit before this check runs. + fn finish_on_timestamp_change( + &mut self, + rtp_timestamp: u32, + ) -> Result<(), RtpDepacketizerError> { + let pending = self + .current + .as_ref() + .map(|current| current.rtp_timestamp) + .or_else(|| self.current_frame.as_ref().map(|frame| frame.rtp_timestamp)) + .or_else(|| self.fragment.as_ref().map(|fragment| fragment.rtp_timestamp)) + .or_else(|| self.av1_fragment.as_ref().map(|fragment| fragment.rtp_timestamp)); + let Some(pending) = pending else { + return Ok(()); + }; + if pending == rtp_timestamp { + return Ok(()); + } + if self.fragment.is_some() || self.av1_fragment.is_some() { + // A new frame began while a fragment was open: its end was lost. + self.discard_in_progress(); + self.dropped_access_units += 1; + return Ok(()); + } + self.finish_pending() + } + + /// Completes the pending access unit or frame for the source's codec. + fn finish_pending(&mut self) -> Result<(), RtpDepacketizerError> { + match self.codec { + EncodedVideoCodec::H264 | EncodedVideoCodec::H265 => self.finish_current(), + _ => self.finish_current_frame(), + } + } + + /// Discards all partially assembled state and gates output on the next keyframe. + fn discard_in_progress(&mut self) { + self.current = None; + self.fragment = None; + self.current_frame = None; + self.av1_fragment = None; + self.pending_bytes = 0; + self.awaiting_keyframe = true; + } + + /// Queues a completed access unit, dropping it while loss recovery gates + /// output on the next keyframe. + fn enqueue(&mut self, access_unit: OwnedEncodedAccessUnit) { + if self.awaiting_keyframe { + if access_unit.frame_type != EncodedFrameType::Key { + self.dropped_access_units += 1; + return; + } + self.awaiting_keyframe = false; + } + self.ready.push_back(access_unit); + } + + fn current_mut( + &mut self, + rtp_timestamp: u32, + ) -> Result<&mut PartialAccessUnit, RtpDepacketizerError> { + if self.current.as_ref().is_some_and(|current| current.rtp_timestamp != rtp_timestamp) { + // Unreachable after `finish_on_timestamp_change`; kept as a + // defensive reset. + self.current = None; + self.fragment = None; + } + + if self.current.is_none() { + let timestamp_us = self.timestamp_mapper.map(rtp_timestamp); + self.current = + Some(PartialAccessUnit { rtp_timestamp, timestamp_us, nal_units: Vec::new() }); + } + + self.current.as_mut().ok_or(RtpDepacketizerError::InvalidFragment) + } + + fn current_frame_mut( + &mut self, + rtp_timestamp: u32, + ) -> Result<&mut PartialFrame, RtpDepacketizerError> { + if self.current_frame.as_ref().is_some_and(|current| current.rtp_timestamp != rtp_timestamp) + { + // Unreachable after `finish_on_timestamp_change`; kept as a + // defensive reset. + self.current_frame = None; + self.av1_fragment = None; + } + + if self.current_frame.is_none() { + let timestamp_us = self.timestamp_mapper.map(rtp_timestamp); + self.current_frame = Some(PartialFrame { + rtp_timestamp, + timestamp_us, + payload: Vec::new(), + frame_type: None, + av1_reduced_still_picture_header: None, + }); + } + + self.current_frame.as_mut().ok_or(RtpDepacketizerError::InvalidFragment) + } + + /// Completes the pending VP8/VP9/AV1 frame and queues it. + fn finish_current_frame(&mut self) -> Result<(), RtpDepacketizerError> { + // An open fragment carrying into the next unit undercounts by at + // most one frame's fragment, which the generous cap absorbs. + self.pending_bytes = 0; + let Some(current) = self.current_frame.take() else { + return Ok(()); + }; + if current.payload.is_empty() { + return Ok(()); + } + + let access_unit = OwnedEncodedAccessUnit::new( + self.codec, + current.payload, + current.timestamp_us, + current.frame_type.unwrap_or(EncodedFrameType::Delta), + self.resolution, + ); + self.enqueue(access_unit); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(super) fn rtp_packet( + sequence_number: u16, + timestamp: u32, + marker: bool, + payload: &[u8], + ) -> Vec { + rtp_packet_from(sequence_number, timestamp, marker, 96, 0x1122_3344, payload) + } + + pub(super) fn rtp_packet_from( + sequence_number: u16, + timestamp: u32, + marker: bool, + payload_type: u8, + ssrc: u32, + payload: &[u8], + ) -> Vec { + let mut packet = Vec::with_capacity(12 + payload.len()); + packet.push(0x80); + packet.push(if marker { 0x80 | payload_type } else { payload_type }); + packet.extend_from_slice(&sequence_number.to_be_bytes()); + packet.extend_from_slice(×tamp.to_be_bytes()); + packet.extend_from_slice(&ssrc.to_be_bytes()); + packet.extend_from_slice(payload); + packet + } + + pub(super) fn assembler(codec: EncodedVideoCodec) -> RtpAccessUnitAssembler { + RtpAccessUnitAssembler::new( + codec, + 96, + 90_000, + H26xParameterSets::default(), + VideoResolution::new(640, 480), + ) + .unwrap() + } + + pub(super) fn push_one( + assembler: &mut RtpAccessUnitAssembler, + bytes: &[u8], + ) -> Option { + assembler.push(bytes).unwrap(); + assembler.pop_ready() + } + + #[test] + fn parses_rtp_packet_header() { + let bytes = rtp_packet(7, 90_000, true, &[0x65, 1, 2]); + let packet = RtpPacket::parse(&bytes).unwrap(); + assert!(packet.marker); + assert_eq!(packet.payload_type, 96); + assert_eq!(packet.sequence_number, 7); + assert_eq!(packet.timestamp, 90_000); + assert_eq!(packet.payload, &[0x65, 1, 2]); + } + + #[test] + fn maps_rtp_timestamp_rollover() { + let mut mapper = RtpTimestampMapper::new(90_000, 1_000).unwrap(); + assert_eq!(mapper.map(u32::MAX - 89), 1_000); + assert_eq!(mapper.map(0), 2_000); + } + + #[test] + fn maps_rtp_timestamps_across_multiple_rollovers() { + let mut mapper = RtpTimestampMapper::new(90_000, 0).unwrap(); + let step = 1u32 << 30; + let mut rtp_timestamp = 0u32; + let mut last_us = mapper.map(rtp_timestamp); + for _ in 0..20 { + rtp_timestamp = rtp_timestamp.wrapping_add(step); + let mapped_us = mapper.map(rtp_timestamp); + assert!(mapped_us > last_us, "mapped timestamps must stay monotonic"); + last_us = mapped_us; + } + assert_eq!(last_us, (20i64 << 30) * 1_000_000 / 90_000); + } + + #[test] + fn maps_reordered_rtp_timestamps() { + let mut mapper = RtpTimestampMapper::new(90_000, 1_000).unwrap(); + assert_eq!(mapper.map(9_000), 1_000); + assert_eq!(mapper.map(18_000), 101_000); + // A late packet maps behind the stream without disturbing what follows. + assert_eq!(mapper.map(15_000), 67_666); + assert_eq!(mapper.map(27_000), 201_000); + } + + #[test] + fn rejects_zero_clock_rate() { + assert_eq!( + RtpTimestampMapper::new(0, 0).unwrap_err(), + RtpDepacketizerError::InvalidClockRate + ); + } + + #[test] + fn ignores_other_payload_types() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // An interloping payload type must not count as a sequence gap. + let other = rtp_packet_from(50, 12_000, true, 97, 0x1122_3344, &[0x65, 9]); + let first = rtp_packet(10, 12_000, false, &[0x65, 1, 2]); + let second = rtp_packet(11, 12_000, true, &[0x41, 3]); + + assert!(push_one(&mut assembler, &first).is_none()); + assert!(push_one(&mut assembler, &other).is_none()); + let access_unit = push_one(&mut assembler, &second).unwrap(); + assert_eq!(access_unit.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2, 0, 0, 0, 1, 0x41, 3]); + assert_eq!(assembler.stats().sequence_gaps, 0); + } + + #[test] + fn locks_to_first_ssrc() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let first = rtp_packet_from(10, 12_000, true, 96, 0xaaaa_aaaa, &[0x65, 1]); + let intruder = rtp_packet_from(70, 15_000, true, 96, 0xbbbb_bbbb, &[0x65, 2]); + let second = rtp_packet_from(11, 15_000, true, 96, 0xaaaa_aaaa, &[0x41, 3]); + + assert!(push_one(&mut assembler, &first).unwrap().frame_type == EncodedFrameType::Key); + assert!(push_one(&mut assembler, &intruder).is_none()); + let access_unit = push_one(&mut assembler, &second).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(assembler.stats().sequence_gaps, 0); + } + + #[test] + fn caps_h264_pending_access_unit_bytes() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let chunk = [0xaa; 60_000]; + + // An FU-A start followed by endless continuations at one timestamp: + // the fragment never completes, so buffering must stop at the cap. + let mut payload = vec![0x7c, 0x85]; + payload.extend_from_slice(&chunk); + assert!(push_one(&mut assembler, &rtp_packet(0, 12_000, false, &payload)).is_none()); + + let mut payload = vec![0x7c, 0x05]; + payload.extend_from_slice(&chunk); + let mut sequence_number = 1u16; + while !assembler.stats().awaiting_keyframe { + assert!(sequence_number < 1_000, "the pending byte cap never triggered"); + let packet = rtp_packet(sequence_number, 12_000, false, &payload); + assert!(push_one(&mut assembler, &packet).is_none()); + sequence_number += 1; + } + assert!(assembler.stats().dropped_access_units >= 1); + + // The stream recovers at the next keyframe. + let key = rtp_packet(sequence_number, 15_000, true, &[0x65, 1, 2]); + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + } + + #[test] + fn caps_vp8_pending_frame_bytes() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let chunk = [0xaa; 60_000]; + + let mut payload = vec![0x10, 0x00]; + payload.extend_from_slice(&chunk); + assert!(push_one(&mut assembler, &rtp_packet(0, 12_000, false, &payload)).is_none()); + + let mut payload = vec![0x00]; + payload.extend_from_slice(&chunk); + let mut sequence_number = 1u16; + while !assembler.stats().awaiting_keyframe { + assert!(sequence_number < 1_000, "the pending byte cap never triggered"); + let packet = rtp_packet(sequence_number, 12_000, false, &payload); + assert!(push_one(&mut assembler, &packet).is_none()); + sequence_number += 1; + } + + let key = rtp_packet(sequence_number, 15_000, true, &[0x10, 0x00, 1, 2]); + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + } + + #[test] + fn timestamp_change_completes_marker_less_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + // The producer never sets the marker bit. + let first = rtp_packet(10, 12_000, false, &[0x65, 1, 2]); + let second = rtp_packet(11, 15_000, false, &[0x41, 3, 4]); + let third = rtp_packet(12, 18_000, false, &[0x41, 5, 6]); + + assert!(push_one(&mut assembler, &first).is_none()); + let key = push_one(&mut assembler, &second).unwrap(); + assert_eq!(key.frame_type, EncodedFrameType::Key); + assert_eq!(key.payload.as_ref(), &[0, 0, 0, 1, 0x65, 1, 2]); + + let delta = push_one(&mut assembler, &third).unwrap(); + assert_eq!(delta.frame_type, EncodedFrameType::Delta); + assert_eq!(delta.payload.as_ref(), &[0, 0, 0, 1, 0x41, 3, 4]); + } + + #[test] + fn timestamp_change_with_open_fragment_drops_access_unit() { + let mut assembler = assembler(EncodedVideoCodec::H264); + let start = rtp_packet(10, 12_000, false, &[0x7c, 0x85, 1, 2]); + // The fragment end never arrives; the next frame starts instead. + let next = rtp_packet(11, 15_000, true, &[0x65, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &next).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(assembler.stats().dropped_access_units, 1); + } +} diff --git a/livekit-capture/src/sources/rtsp/rtp/vpx.rs b/livekit-capture/src/sources/rtsp/rtp/vpx.rs new file mode 100644 index 000000000..d96511d3e --- /dev/null +++ b/livekit-capture/src/sources/rtsp/rtp/vpx.rs @@ -0,0 +1,427 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! VP8 (RFC 7741) and VP9 (draft-ietf-payload-vp9) RTP payload handling. +//! +//! Only single-layer streams are supported; scalable streams are rejected +//! with [`RtpDepacketizerError::UnsupportedPayloadDescriptor`]. + +use super::{RtpAccessUnitAssembler, RtpDepacketizerError, RtpPacket}; +use crate::{encoded::EncodedFrameType, sources::rtsp::bits::ByteReader}; + +impl RtpAccessUnitAssembler { + pub(super) fn push_vp8_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_vp8_payload_descriptor(packet.payload)?; + if descriptor.payload.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + + let frame = self.current_frame_mut(packet.timestamp)?; + if frame.payload.is_empty() { + if !descriptor.start_of_partition || descriptor.partition_id != 0 { + // The beginning of this frame was lost. + self.discard_in_progress(); + return Ok(()); + } + frame.frame_type = Some(if is_vp8_keyframe(descriptor.payload) { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }); + } + frame.payload.extend_from_slice(descriptor.payload); + Ok(()) + } + + pub(super) fn push_vp9_payload( + &mut self, + packet: &RtpPacket<'_>, + ) -> Result<(), RtpDepacketizerError> { + let descriptor = parse_vp9_payload_descriptor(packet.payload)?; + if descriptor.payload.is_empty() { + return Err(RtpDepacketizerError::UnsupportedPayload); + } + if descriptor.spatial_id.unwrap_or(0) != 0 + || descriptor.inter_layer_predicted.unwrap_or(false) + { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + let frame = self.current_frame_mut(packet.timestamp)?; + if frame.payload.is_empty() { + if !descriptor.beginning_of_frame { + // The beginning of this frame was lost. + self.discard_in_progress(); + return Ok(()); + } + frame.frame_type = Some( + if !descriptor.inter_picture_predicted || is_vp9_keyframe(descriptor.payload) { + EncodedFrameType::Key + } else { + EncodedFrameType::Delta + }, + ); + } + frame.payload.extend_from_slice(descriptor.payload); + Ok(()) + } +} + +#[derive(Debug, Clone, Copy)] +struct Vp8PayloadDescriptor<'a> { + start_of_partition: bool, + partition_id: u8, + payload: &'a [u8], +} + +#[derive(Debug, Clone, Copy)] +struct Vp9PayloadDescriptor<'a> { + beginning_of_frame: bool, + inter_picture_predicted: bool, + spatial_id: Option, + inter_layer_predicted: Option, + payload: &'a [u8], +} + +fn parse_vp8_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let descriptor = reader.get_u8().ok_or_else(malformed)?; + let start_of_partition = descriptor & 0x10 != 0; + let partition_id = descriptor & 0x0f; + if descriptor & 0x80 != 0 { + let extension = reader.get_u8().ok_or_else(malformed)?; + if extension & 0x80 != 0 { + let picture_id = reader.get_u8().ok_or_else(malformed)?; + if picture_id & 0x80 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } + } + if extension & 0x40 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } + if extension & 0x20 != 0 || extension & 0x10 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } + } + Ok(Vp8PayloadDescriptor { start_of_partition, partition_id, payload: reader.take_rest() }) +} + +fn parse_vp9_payload_descriptor( + payload: &[u8], +) -> Result, RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let mut reader = ByteReader::new(payload); + + let descriptor = reader.get_u8().ok_or_else(malformed)?; + if descriptor & 0x10 != 0 { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + let beginning_of_frame = descriptor & 0x08 != 0; + let inter_picture_predicted = descriptor & 0x40 != 0; + if descriptor & 0x80 != 0 { + let picture_id = reader.get_u8().ok_or_else(malformed)?; + if picture_id & 0x80 != 0 { + reader.skip(1).ok_or_else(malformed)?; + } + } + + let mut spatial_id = None; + let mut inter_layer_predicted = None; + if descriptor & 0x20 != 0 { + let layer_info = reader.get_u8().ok_or_else(malformed)?; + spatial_id = Some((layer_info >> 1) & 0x07); + inter_layer_predicted = Some(layer_info & 0x01 != 0); + reader.skip(1).ok_or_else(malformed)?; // TL0PICIDX in non-flexible mode + } + + if descriptor & 0x02 != 0 { + skip_vp9_scalability_structure(&mut reader)?; + } + + Ok(Vp9PayloadDescriptor { + beginning_of_frame, + inter_picture_predicted, + spatial_id, + inter_layer_predicted, + payload: reader.take_rest(), + }) +} + +fn skip_vp9_scalability_structure( + reader: &mut ByteReader<'_>, +) -> Result<(), RtpDepacketizerError> { + let malformed = || RtpDepacketizerError::UnsupportedPayload; + let structure = reader.get_u8().ok_or_else(malformed)?; + + let spatial_layers = ((structure >> 5) & 0x07) + 1; + if spatial_layers != 1 { + return Err(RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + if structure & 0x10 != 0 { + reader.skip(usize::from(spatial_layers) * 4).ok_or_else(malformed)?; + } + + if structure & 0x08 != 0 { + let group_count = reader.get_u8().ok_or_else(malformed)?; + for _ in 0..group_count { + let group = reader.get_u8().ok_or_else(malformed)?; + reader.skip(usize::from((group >> 2) & 0x03)).ok_or_else(malformed)?; + } + } + + Ok(()) +} + +fn is_vp8_keyframe(payload: &[u8]) -> bool { + payload.first().is_some_and(|header| header & 0x01 == 0) +} + +/// Parses the start of a VP9 uncompressed frame header, whose `f(n)` fields +/// are MSB-first, and reports whether it begins a keyframe. +fn is_vp9_keyframe(payload: &[u8]) -> bool { + let Some(&first_byte) = payload.first() else { + return false; + }; + // frame_marker: f(2), must be 0b10. + if first_byte >> 6 != 0b10 { + return false; + } + + let mut bit_offset = 2usize; + let profile_low = read_bit(first_byte, bit_offset); + bit_offset += 1; + let profile_high = read_bit(first_byte, bit_offset); + bit_offset += 1; + let profile = profile_low | (profile_high << 1); + if profile == 3 { + bit_offset += 1; // reserved_zero + } + // show_existing_frame: a repeated frame is never a keyframe. + if read_bit(first_byte, bit_offset) != 0 { + return false; + } + bit_offset += 1; + // frame_type: 0 is KEY_FRAME. + read_bit(first_byte, bit_offset) == 0 +} + +/// Reads bit `bit_offset` of `byte`, counting from the most significant bit. +fn read_bit(byte: u8, bit_offset: usize) -> u8 { + (byte >> (7 - bit_offset)) & 0x01 +} + +#[cfg(test)] +mod tests { + use super::super::tests::{assembler, push_one, rtp_packet}; + use super::*; + use crate::encoded::EncodedVideoCodec; + + #[test] + fn assembles_vp8_fragments() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let start = rtp_packet(10, 12_000, false, &[0x10, 0x00, 1, 2]); + let end = rtp_packet(11, 12_000, true, &[0x00, 3, 4]); + + assert!(push_one(&mut assembler, &start).is_none()); + let access_unit = push_one(&mut assembler, &end).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 1, 2, 3, 4]); + } + + #[test] + fn drops_vp8_mid_frame_start() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let mid_frame = rtp_packet(10, 12_000, true, &[0x00, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x10, 0x00, 3, 4]); + + assert!(push_one(&mut assembler, &mid_frame).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 3, 4]); + } + + #[test] + fn sequence_gap_recovers_vp8_at_next_keyframe() { + let mut assembler = assembler(EncodedVideoCodec::VP8); + let start = rtp_packet(10, 12_000, false, &[0x10, 0x00, 1, 2]); + let delta = rtp_packet(12, 15_000, true, &[0x10, 0x01, 3, 4]); + let key = rtp_packet(13, 18_000, true, &[0x10, 0x00, 5, 6]); + + assert!(push_one(&mut assembler, &start).is_none()); + // The gap dropped the fragment; the delta frame after it is withheld. + assert!(push_one(&mut assembler, &delta).is_none()); + let stats = assembler.stats(); + assert_eq!(stats.sequence_gaps, 1); + assert_eq!(stats.dropped_access_units, 1); + assert!(stats.awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x00, 5, 6]); + assert!(!assembler.stats().awaiting_keyframe); + } + + #[test] + fn assembles_vp9_single_layer_frame() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x0c, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_non_flexible_layer_descriptor() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x2c, 0x10, 7, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_single_layer_scalability_structure() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet( + 10, + 12_000, + true, + &[ + 0x0e, // B, E, V + 0x18, // one spatial layer, resolution present, picture group present + 0x01, 0x40, 0x00, 0xb4, // 320x180 + 0x01, // one picture group + 0x04, // one reference index + 0x01, // P_DIFF + 0x82, 1, 2, + ], + ); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn assembles_vp9_descriptor_keyframe_from_prediction_bit() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet( + 10, + 12_000, + true, + &[ + 0x0e, // B, E, V; P is clear, so this is not inter-picture predicted. + 0x18, // one spatial layer, resolution present, picture group present + 0x02, 0x80, 0x01, 0x68, // 640x360 + 0x01, // one picture group + 0x04, // one reference index + 0x01, // P_DIFF + 0xb1, 1, 2, + ], + ); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0xb1, 1, 2]); + } + + #[test] + fn assembles_vp9_predicted_frame_as_delta() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + // P is set and the payload is an inter frame: must not classify as Key. + let packet = rtp_packet(10, 12_000, true, &[0x4c, 0x86, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Delta); + assert_eq!(access_unit.payload.as_ref(), &[0x86, 1, 2]); + } + + #[test] + fn vp9_bitstream_keyframe_overrides_predicted_bit() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + // P is set but the uncompressed header says KEY_FRAME. + let packet = rtp_packet(10, 12_000, true, &[0x4c, 0x82, 1, 2]); + + let access_unit = push_one(&mut assembler, &packet).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 1, 2]); + } + + #[test] + fn classifies_vp9_uncompressed_header_frame_types() { + // 0b1000_0010: marker, profile 0, show_existing=0, KEY_FRAME, show_frame=1. + assert!(is_vp9_keyframe(&[0x82])); + // 0b1000_0011: keyframe with error_resilient_mode set. + assert!(is_vp9_keyframe(&[0x83])); + // 0b1011_0000: profile 3 keyframe. + assert!(is_vp9_keyframe(&[0xb0])); + // 0b1000_0110: frame_type=1, an inter frame. + assert!(!is_vp9_keyframe(&[0x86])); + // 0b1011_0010: profile 3 inter frame. + assert!(!is_vp9_keyframe(&[0xb2])); + // 0b1000_1000: show_existing_frame repeats a decoded frame. + assert!(!is_vp9_keyframe(&[0x88])); + // 0b0000_0010: invalid frame_marker. + assert!(!is_vp9_keyframe(&[0x02])); + assert!(!is_vp9_keyframe(&[])); + } + + #[test] + fn rejects_vp9_multi_layer_scalability_structure() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x0e, 0x20, 0x82, 1, 2]); + + let err = assembler.push(&packet).unwrap_err(); + assert_eq!(err, RtpDepacketizerError::UnsupportedPayloadDescriptor); + } + + #[test] + fn drops_vp9_mid_frame_start() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let mid_frame = rtp_packet(10, 12_000, true, &[0x04, 0x82, 1, 2]); + let key = rtp_packet(11, 15_000, true, &[0x0c, 0x82, 3, 4]); + + assert!(push_one(&mut assembler, &mid_frame).is_none()); + assert!(assembler.stats().awaiting_keyframe); + + let access_unit = push_one(&mut assembler, &key).unwrap(); + assert_eq!(access_unit.frame_type, EncodedFrameType::Key); + assert_eq!(access_unit.payload.as_ref(), &[0x82, 3, 4]); + } + + #[test] + fn rejects_vp9_flexible_mode() { + let mut assembler = assembler(EncodedVideoCodec::VP9); + let packet = rtp_packet(10, 12_000, true, &[0x1c, 0xa2, 1, 2]); + + let err = assembler.push(&packet).unwrap_err(); + assert_eq!(err, RtpDepacketizerError::UnsupportedPayloadDescriptor); + } +} diff --git a/livekit-capture/src/sources/rtsp/sdp.rs b/livekit-capture/src/sources/rtsp/sdp.rs new file mode 100644 index 000000000..ca809d0b1 --- /dev/null +++ b/livekit-capture/src/sources/rtsp/sdp.rs @@ -0,0 +1,558 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! SDP parsing for the video track of an RTSP DESCRIBE response. + +use base64::{engine::general_purpose, Engine as _}; + +use super::{rtp::H26xParameterSets, RtspVideoSourceError}; +use crate::{encoded::EncodedVideoCodec, primitive::VideoResolution}; + +/// RTP timestamp clock rate assumed when the `rtpmap` omits one. +const DEFAULT_CLOCK_RATE: u32 = 90_000; + +/// The video stream selected from an SDP session description. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SdpSession { + /// Selected video track. + pub(super) video: SdpVideoTrack, + /// Aggregate control URL for session-level requests (PLAY, OPTIONS, + /// TEARDOWN), from the session-level `a=control` attribute. + pub(super) aggregate_control_url: String, +} + +/// A video track selected from the SDP. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct SdpVideoTrack { + /// RTP payload codec. + pub(super) codec: EncodedVideoCodec, + /// RTP payload type. + pub(super) payload_type: u8, + /// RTP timestamp clock rate. + pub(super) clock_rate: u32, + /// Media control URL used for SETUP. + pub(super) control_url: String, + /// Out-of-band parameter sets from the track's `fmtp` attribute. + pub(super) parameter_sets: H26xParameterSets, + /// Resolution from the track's `a=framesize` attribute, when present. + pub(super) framesize: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct SdpRtpMap { + payload_type: u8, + codec: EncodedVideoCodec, + clock_rate: u32, +} + +/// Selects a video track from an SDP session description. +/// +/// `base_url` is the URL relative control attributes resolve against: the +/// DESCRIBE response's `Content-Base` when present, else the request URL. +/// With an `expected_codec`, only tracks carrying that codec match; without +/// one, the first supported video track is selected. +pub(super) fn parse_sdp_session( + base_url: &str, + sdp: &[u8], + expected_codec: Option, +) -> Result { + let session = sdp_types::Session::parse(sdp).map_err(|err| { + // The parser error can quote server-provided bytes; escape them. + log::debug!("failed to parse SDP: {}", err.to_string().escape_debug()); + RtspVideoSourceError::InvalidSdp + })?; + let session_control = attribute_value(&session.attributes, "control"); + + // Selecting among multiple video tracks is not supported (only ordinal + // selection would have standard SDP footing); surface the choice so it + // is not hidden. + let video_tracks = || session.medias.iter().filter(|media| media.media == "video"); + let video_track_count = video_tracks().count(); + if video_track_count > 1 { + let summary = video_tracks() + .map(|media| { + let codecs: Vec<&str> = attribute_values(&media.attributes, "rtpmap") + .filter_map(|rtpmap| rtpmap.split_whitespace().nth(1)) + .filter_map(|encoding| encoding.split('/').next()) + .collect(); + if codecs.is_empty() { "?".to_owned() } else { codecs.join("+") } + }) + .collect::>() + .join(", "); + log::info!( + "RTSP SDP offers {video_track_count} video tracks ({}); \ + using the first one with a supported codec", + super::sanitized(summary), + ); + } + + let mut offered = Vec::new(); + for media in &session.medias { + if media.media != "video" { + continue; + } + let rtp_maps: Vec = attribute_values(&media.attributes, "rtpmap") + .filter_map(parse_rtpmap) + .collect(); + + for payload_type in media.fmt.split_whitespace().filter_map(|pt| pt.parse::().ok()) { + let Some(rtp_map) = rtp_maps.iter().find(|map| map.payload_type == payload_type) + else { + continue; + }; + if let Some(expected) = expected_codec { + if rtp_map.codec != expected { + if !offered.contains(&rtp_map.codec) { + offered.push(rtp_map.codec); + } + continue; + } + } + + let parameter_sets = attribute_values(&media.attributes, "fmtp") + .find_map(|value| { + let (fmtp_payload_type, params) = + value.trim().split_once(char::is_whitespace)?; + (fmtp_payload_type.parse::().ok()? == payload_type) + .then(|| parse_fmtp_parameter_sets(rtp_map.codec, params)) + }) + .unwrap_or_default(); + let framesize = attribute_values(&media.attributes, "framesize") + .filter_map(parse_framesize) + .find_map(|(framesize_payload_type, resolution)| { + (framesize_payload_type == payload_type).then_some(resolution) + }); + + return Ok(SdpSession { + video: SdpVideoTrack { + codec: rtp_map.codec, + payload_type, + clock_rate: rtp_map.clock_rate, + control_url: resolve_control_url( + base_url, + attribute_value(&media.attributes, "control"), + ), + parameter_sets, + framesize, + }, + aggregate_control_url: resolve_control_url(base_url, session_control), + }); + } + } + + match expected_codec { + Some(expected) if !offered.is_empty() => { + Err(RtspVideoSourceError::CodecMismatch { expected, offered }) + } + _ => Err(RtspVideoSourceError::MissingVideoTrack), + } +} + +/// Returns the first value of the named attribute. +fn attribute_value<'a>(attributes: &'a [sdp_types::Attribute], name: &'a str) -> Option<&'a str> { + attribute_values(attributes, name).next() +} + +/// Returns every value of the named attribute. +fn attribute_values<'a>( + attributes: &'a [sdp_types::Attribute], + name: &'a str, +) -> impl Iterator + 'a { + attributes + .iter() + .filter(move |attribute| attribute.attribute == name) + .filter_map(|attribute| attribute.value.as_deref()) + .map(str::trim) +} + +fn parse_rtpmap(rtpmap: &str) -> Option { + let (payload_type, encoding) = rtpmap.trim().split_once(' ')?; + let payload_type = payload_type.parse().ok()?; + let mut encoding_parts = encoding.split('/'); + let codec_name = encoding_parts.next()?; + let codec = parse_sdp_codec(codec_name)?; + let clock_rate = encoding_parts + .next() + .and_then(|clock_rate| clock_rate.parse().ok()) + .unwrap_or(DEFAULT_CLOCK_RATE); + Some(SdpRtpMap { payload_type, codec, clock_rate }) +} + +fn parse_sdp_codec(codec_name: &str) -> Option { + if codec_name.eq_ignore_ascii_case("H264") { + Some(EncodedVideoCodec::H264) + } else if codec_name.eq_ignore_ascii_case("H265") || codec_name.eq_ignore_ascii_case("HEVC") { + Some(EncodedVideoCodec::H265) + } else if codec_name.eq_ignore_ascii_case("VP8") { + Some(EncodedVideoCodec::VP8) + } else if codec_name.eq_ignore_ascii_case("VP9") { + Some(EncodedVideoCodec::VP9) + } else if codec_name.eq_ignore_ascii_case("AV1") { + Some(EncodedVideoCodec::AV1) + } else { + None + } +} + +/// Parses an `a=framesize: -` value (RFC 6064). +fn parse_framesize(framesize: &str) -> Option<(u8, VideoResolution)> { + let (payload_type, dimensions) = framesize.trim().split_once(char::is_whitespace)?; + let payload_type = payload_type.parse().ok()?; + let (width, height) = dimensions.trim().split_once('-')?; + let resolution = VideoResolution::new(width.parse().ok()?, height.parse().ok()?); + Some((payload_type, resolution)) +} + +/// Decodes out-of-band parameter sets from an `fmtp` parameter list: +/// `sprop-parameter-sets` for H.264, `sprop-vps`/`sprop-sps`/`sprop-pps` +/// for H.265. Individually malformed entries are skipped. +fn parse_fmtp_parameter_sets(codec: EncodedVideoCodec, params: &str) -> H26xParameterSets { + let mut sets = H26xParameterSets::default(); + for param in params.split(';') { + let Some((name, value)) = param.trim().split_once('=') else { + continue; + }; + match codec { + EncodedVideoCodec::H264 if name.eq_ignore_ascii_case("sprop-parameter-sets") => { + for nal in decode_base64_nals(value) { + // Classify by NAL type rather than position: the + // attribute usually lists SPS then PPS, but not always. + match nal.first().map(|header| header & 0x1f) { + Some(7) => sets.sps.push(nal), + Some(8) => sets.pps.push(nal), + _ => {} + } + } + } + EncodedVideoCodec::H265 => { + let target = if name.eq_ignore_ascii_case("sprop-vps") { + &mut sets.vps + } else if name.eq_ignore_ascii_case("sprop-sps") { + &mut sets.sps + } else if name.eq_ignore_ascii_case("sprop-pps") { + &mut sets.pps + } else { + continue; + }; + target.extend(decode_base64_nals(value)); + } + _ => {} + } + } + sets +} + +fn decode_base64_nals(value: &str) -> Vec> { + value + .split(',') + .map(str::trim) + .filter(|encoded| !encoded.is_empty()) + .filter_map(|encoded| general_purpose::STANDARD.decode(encoded).ok()) + .filter(|nal| !nal.is_empty()) + .collect() +} + +/// Splits an `rtsp://` or `rtsps://` URL into its scheme and remainder. +fn split_rtsp_scheme(url: &str) -> Option<(&str, &str)> { + if let Some(rest) = url.strip_prefix("rtsp://") { + return Some(("rtsp://", rest)); + } + url.strip_prefix("rtsps://").map(|rest| ("rtsps://", rest)) +} + +/// Resolves an SDP `control` attribute against the session base URL. +fn resolve_control_url(base_url: &str, control: Option<&str>) -> String { + let Some(control) = control.map(str::trim).filter(|control| !control.is_empty()) else { + return base_url.to_owned(); + }; + if control == "*" { + return base_url.to_owned(); + } + if split_rtsp_scheme(control).is_some() { + return control.to_owned(); + } + if control.starts_with('/') { + // An absolute path keeps the base URL's scheme and authority. + let (scheme, rest) = split_rtsp_scheme(base_url).unwrap_or(("rtsp://", base_url)); + let authority = rest.split('/').next().unwrap_or(rest); + return format!("{scheme}{authority}{control}"); + } + format!("{}/{}", base_url.trim_end_matches('/'), control) +} + +#[cfg(test)] +mod tests { + use super::*; + + const BASE_URL: &str = "rtsp://camera.example/live"; + + #[test] + fn parses_sdp_video_track() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.payload_type, 96); + assert_eq!(session.video.clock_rate, 90_000); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=1"); + assert_eq!(session.aggregate_control_url, BASE_URL); + assert!(session.video.parameter_sets.is_empty()); + assert_eq!(session.video.framesize, None); + } + + #[test] + fn parses_vp8_vp9_and_av1_sdp_video_tracks() { + for (rtpmap, codec) in [ + ("VP8/90000", EncodedVideoCodec::VP8), + ("VP9/90000", EncodedVideoCodec::VP9), + ("AV1/90000", EncodedVideoCodec::AV1), + ] { + let sdp = format!( + "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 {rtpmap}\r\n" + ); + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(codec)).unwrap(); + + assert_eq!(session.video.codec, codec); + assert_eq!(session.video.payload_type, 96); + assert_eq!(session.video.clock_rate, 90_000); + } + } + + #[test] + fn rejects_sdp_codec_mismatch() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 VP9/90000\r\n"; + + let err = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::AV1)).unwrap_err(); + + match err { + RtspVideoSourceError::CodecMismatch { expected, offered } => { + assert_eq!(expected, EncodedVideoCodec::AV1); + assert_eq!(offered, vec![EncodedVideoCodec::VP9]); + } + other => panic!("expected codec mismatch, got {other:?}"), + } + } + + #[test] + fn selects_expected_codec_among_multiple_payload_types() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.payload_type, 96); + } + + #[test] + fn selects_expected_codec_from_later_video_section() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=2\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::H264)).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); + } + + #[test] + fn rejects_sdp_listing_all_offered_codecs_when_none_match() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 98 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:98 H265/90000\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let err = parse_sdp_session(BASE_URL, sdp.as_bytes(), Some(EncodedVideoCodec::VP8)).unwrap_err(); + + match err { + RtspVideoSourceError::CodecMismatch { expected, offered } => { + assert_eq!(expected, EncodedVideoCodec::VP8); + assert_eq!(offered, vec![EncodedVideoCodec::H265, EncodedVideoCodec::H264]); + } + other => panic!("expected codec mismatch, got {other:?}"), + } + } + + #[test] + fn ignores_audio_sections() { + let sdp = "\ +v=0\r\n\ +m=audio 0 RTP/AVP 97\r\n\ +a=rtpmap:97 PCMU/8000\r\n\ +a=control:trackID=1\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=2\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.codec, EncodedVideoCodec::H264); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=2"); + } + + #[test] + fn parses_h264_sprop_parameter_sets() { + // 0x67 (SPS) and 0x68 (PPS) prefixed NAL units. + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=fmtp:96 packetization-mode=1;sprop-parameter-sets=ZwlA,aAlB;profile-level-id=42e01e\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); + assert_eq!(session.video.parameter_sets.pps, vec![vec![0x68, 0x09, 0x41]]); + assert!(session.video.parameter_sets.vps.is_empty()); + } + + #[test] + fn parses_h265_sprop_attributes() { + // 0x40 (VPS), 0x42 (SPS), and 0x44 (PPS) prefixed NAL units. + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H265/90000\r\n\ +a=fmtp:96 sprop-vps=QAEB;sprop-sps=QgEC;sprop-pps=RAED\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.parameter_sets.vps, vec![vec![0x40, 0x01, 0x01]]); + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x42, 0x01, 0x02]]); + assert_eq!(session.video.parameter_sets.pps, vec![vec![0x44, 0x01, 0x03]]); + } + + #[test] + fn tolerates_malformed_sprop_entries() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=fmtp:96 sprop-parameter-sets=!!!not-base64!!!,ZwlA\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.parameter_sets.sps, vec![vec![0x67, 0x09, 0x40]]); + assert!(session.video.parameter_sets.pps.is_empty()); + } + + #[test] + fn parses_framesize_attribute() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=rtpmap:96 H264/90000\r\n\ +a=framesize:96 1280-720\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.framesize, Some(VideoResolution::new(1280, 720))); + } + + #[test] + fn resolves_session_level_aggregate_control() { + let sdp = "\ +v=0\r\n\ +a=control:rtsp://camera.example/live/aggregate\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.aggregate_control_url, "rtsp://camera.example/live/aggregate"); + assert_eq!(session.video.control_url, "rtsp://camera.example/live/trackID=1"); + } + + #[test] + fn star_control_resolves_to_base_url() { + let sdp = "\ +v=0\r\n\ +a=control:*\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = parse_sdp_session(BASE_URL, sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.aggregate_control_url, BASE_URL); + } + + #[test] + fn resolves_absolute_path_control_url() { + assert_eq!( + resolve_control_url(BASE_URL, Some("/stream/trackID=1")), + "rtsp://camera.example/stream/trackID=1" + ); + } + + #[test] + fn resolves_control_urls_with_rtsps_scheme() { + // An absolute path keeps the rtsps scheme of the base URL. + assert_eq!( + resolve_control_url("rtsps://camera.example:7441/live", Some("/stream/trackID=1")), + "rtsps://camera.example:7441/stream/trackID=1" + ); + // A relative control appends to the rtsps base. + assert_eq!( + resolve_control_url("rtsps://camera.example:7441/live", Some("trackID=1")), + "rtsps://camera.example:7441/live/trackID=1" + ); + // An absolute rtsps URL passes through. + assert_eq!( + resolve_control_url(BASE_URL, Some("rtsps://camera.example/other")), + "rtsps://camera.example/other" + ); + } + + #[test] + fn resolves_control_against_content_base() { + let sdp = "\ +v=0\r\n\ +m=video 0 RTP/AVP 96\r\n\ +a=control:trackID=1\r\n\ +a=rtpmap:96 H264/90000\r\n"; + + let session = + parse_sdp_session("rtsp://camera.example/relocated/", sdp.as_bytes(), None).unwrap(); + + assert_eq!(session.video.control_url, "rtsp://camera.example/relocated/trackID=1"); + } +} diff --git a/livekit-capture/tests/README.md b/livekit-capture/tests/README.md new file mode 100644 index 000000000..a0ba4b40e --- /dev/null +++ b/livekit-capture/tests/README.md @@ -0,0 +1,50 @@ +# Integration Tests + +Integration tests verify capture sources against real backends, so they are +not enabled by default. Each source has an internal test feature named +`__test-source-` that builds and runs its tests in +`tests/source__test.rs`. Tests exercise the source directly through +its public API (construction and `next_access_unit`); they do not publish to +an RTC source and need no LiveKit server. + +Some sources can only be tested on hosts that provide their backend (for +example, a future `__test-source-device` needs a capture device), so each +section below documents its prerequisites. + +## RTSP (`__test-source-rtsp`) + +Each test starts an in-process GStreamer RTSP server on an ephemeral +localhost port and streams real encoded video to the source. The system +GStreamer installation must include the RTSP server library, the x264, +x265, VP8/VP9, and AV1 encoder plugins, and the RTP payloaders from +gst-plugins-rs (`rtpav1pay`): + +- macOS: `brew install gstreamer` +- Debian/Ubuntu: `apt install libgstrtspserver-1.0-dev + gstreamer1.0-plugins-good gstreamer1.0-plugins-bad + gstreamer1.0-plugins-ugly` (in addition to the base development packages) + +```sh +cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test +``` + +The RTSPS tests generate a self-signed certificate at run time (via +`rcgen`), so TLS needs no host setup beyond the GStreamer packages above. + +## Logging + +Tests use the [`test-log`](https://crates.io/crates/test-log) `#[test]` +attribute, so `log::` output from the sources and the test helpers is +recorded per test. The standard test harness only prints it for *failing* +tests; to see it for passing tests, disable output capture: + +```sh +cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test -- --nocapture +``` + +That shows info-level logs (stream setup, discovered settings). For the +per-access-unit trace, raise the filter: + +```sh +RUST_LOG=debug cargo test -p livekit-capture --features __test-source-rtsp --test source_rtsp_test -- --nocapture +``` diff --git a/livekit-capture/tests/common/mod.rs b/livekit-capture/tests/common/mod.rs new file mode 100644 index 000000000..79b5acbd0 --- /dev/null +++ b/livekit-capture/tests/common/mod.rs @@ -0,0 +1,64 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Shared helpers for the livekit-capture integration tests. +//! +//! Source-specific helpers (test servers, pipelines) live in a submodule +//! per source, gated by that source's test feature. + +#[cfg(feature = "__test-source-rtsp")] +pub mod rtsp; + +use std::time::{Duration, Instant}; + +use livekit_capture::{ + encoded::{EncodedVideoSource, OwnedEncodedAccessUnit}, + pump::PumpStop, +}; + +/// How long one test waits for its access units before failing. +const PULL_DEADLINE: Duration = Duration::from_secs(15); + +/// Pulls `count` access units from an encoded source, failing the test if +/// the stream errors, ends, or stalls. +pub fn pull_access_units( + source: &mut impl EncodedVideoSource, + count: usize, +) -> Vec { + let stop = PumpStop::new(); + let deadline = Instant::now() + PULL_DEADLINE; + let mut access_units = Vec::with_capacity(count); + while access_units.len() < count { + assert!( + Instant::now() < deadline, + "timed out after {} of {count} access units", + access_units.len() + ); + match source.next_access_unit(&stop).expect("source failed") { + Some(access_unit) => { + log::debug!( + "pulled access unit {}/{count}: {:?} {:?}, {} bytes, ts {}us", + access_units.len() + 1, + access_unit.codec, + access_unit.frame_type, + access_unit.payload.len(), + access_unit.timestamp_us, + ); + access_units.push(access_unit); + } + None => panic!("stream ended after {} of {count} access units", access_units.len()), + } + } + access_units +} diff --git a/livekit-capture/tests/common/rtsp.rs b/livekit-capture/tests/common/rtsp.rs new file mode 100644 index 000000000..5204b73b5 --- /dev/null +++ b/livekit-capture/tests/common/rtsp.rs @@ -0,0 +1,271 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! RTSP test helpers: an in-process GStreamer RTSP server and the encoder +//! pipelines it serves. + +use std::{ + thread, + time::{Duration, Instant}, +}; + +use gstreamer::{self as gst, glib}; +use gstreamer_rtsp_server::{prelude::*, RTSPAuth, RTSPMediaFactory, RTSPServer, RTSPToken}; +use livekit_capture::{primitive::VideoResolution, sources::rtsp::RtspVideoSourceConfig}; + +/// Default test streams are 640x480 at 30 fps with a keyframe every 30 +/// frames. +pub const TEST_RESOLUTION: VideoResolution = VideoResolution::new(640, 480); + +/// Codecs the test server can encode. +#[derive(Debug, Clone, Copy)] +pub enum TestCodec { + H264, + H265, + Vp8, + Vp9, + Av1, +} + +/// Builds an encoder pipeline for the test server at the given resolution. +pub fn pipeline(codec: TestCodec, resolution: VideoResolution) -> String { + let encode = match codec { + TestCodec::H264 => { + "x264enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + byte-stream=true aud=true \ + ! h264parse config-interval=-1 ! rtph264pay name=pay0 pt=96 config-interval=1" + } + TestCodec::H265 => { + "x265enc tune=zerolatency speed-preset=ultrafast key-int-max=30 bitrate=500 \ + option-string=repeat-headers=1:aud=1:open-gop=0 \ + ! h265parse config-interval=-1 ! rtph265pay name=pay0 pt=96 config-interval=1" + } + TestCodec::Vp8 => { + "vp8enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500000 ! rtpvp8pay name=pay0 pt=96" + } + TestCodec::Vp9 => { + "vp9enc deadline=1 cpu-used=8 keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500000 ! rtpvp9pay name=pay0 pt=96" + } + TestCodec::Av1 => { + "av1enc cpu-used=8 usage-profile=realtime keyframe-max-dist=30 lag-in-frames=0 \ + target-bitrate=500 ! av1parse \ + ! video/x-av1,stream-format=obu-stream,alignment=tu ! rtpav1pay name=pay0 pt=96" + } + }; + format!( + "videotestsrc is-live=true \ + ! video/x-raw,width={},height={},framerate=30/1 ! videoconvert ! {encode}", + resolution.width, resolution.height, + ) +} + +/// Builds a pipeline at the default resolution. +pub fn default_pipeline(codec: TestCodec) -> String { + pipeline(codec, TEST_RESOLUTION) +} + +/// Builds a two-track pipeline: video at the default resolution plus a PCMA +/// audio track, mirroring the SDP shape of a typical camera. +pub fn default_pipeline_with_audio(codec: TestCodec) -> String { + format!( + "{} audiotestsrc is-live=true ! alawenc ! rtppcmapay name=pay1 pt=8", + default_pipeline(codec), + ) +} + +/// An in-process GStreamer RTSP server serving one launch pipeline at +/// `/test` on an ephemeral localhost port. +pub struct RtspTestServer { + main_loop: glib::MainLoop, + thread: Option>, + port: i32, + tls: bool, +} + +/// Authentication required by a test server. +#[derive(Debug, Clone, Copy)] +enum TestAuth<'a> { + None, + Basic { username: &'a str, password: &'a str }, + Digest { username: &'a str, password: &'a str }, +} + +impl RtspTestServer { + /// Starts a server streaming `media_pipeline`, which must contain an RTP + /// payloader named `pay0`. + pub fn launch(media_pipeline: &str) -> Self { + Self::launch_inner(media_pipeline, TestAuth::None, false) + } + + /// Starts a server that requires Basic authentication. + pub fn launch_with_basic_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, TestAuth::Basic { username, password }, false) + } + + /// Starts a server that requires Digest authentication. + pub fn launch_with_digest_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, TestAuth::Digest { username, password }, false) + } + + /// Starts a server that requires TLS (`rtsps://`), presenting a + /// freshly generated self-signed certificate. + pub fn launch_tls(media_pipeline: &str) -> Self { + Self::launch_inner(media_pipeline, TestAuth::None, true) + } + + /// Starts a server that requires both TLS and Digest authentication. + pub fn launch_tls_with_digest_auth( + media_pipeline: &str, + username: &str, + password: &str, + ) -> Self { + Self::launch_inner(media_pipeline, TestAuth::Digest { username, password }, true) + } + + fn launch_inner(media_pipeline: &str, test_auth: TestAuth<'_>, tls: bool) -> Self { + gst::init().expect("failed to initialize GStreamer"); + + // Each server runs on its own main context so parallel tests never + // contend for the default one. + let context = glib::MainContext::new(); + let server = RTSPServer::new(); + server.set_address("127.0.0.1"); + // Bind an ephemeral port; the real port is read back after attach. + server.set_service("0"); + + let factory = RTSPMediaFactory::new(); + factory.set_launch(&format!("( {media_pipeline} )")); + factory.set_shared(false); + + if !matches!(test_auth, TestAuth::None) || tls { + factory.add_role_from_structure( + &gst::Structure::builder("user") + .field("media.factory.access", true) + .field("media.factory.construct", true) + .build(), + ); + let auth = RTSPAuth::new(); + if tls { + // Once a certificate is set, gst-rtsp-server requires TLS on + // every connection to this server instance. + auth.set_tls_certificate(Some(&self_signed_certificate())); + } + let token = RTSPToken::builder().field("media.factory.role", "user").build(); + match test_auth { + TestAuth::None => { + // TLS without authentication: admit anonymous clients. + let mut token = token; + auth.set_default_token(Some(&mut token)); + } + TestAuth::Basic { username, password } => { + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Basic); + auth.add_basic(&RTSPAuth::make_basic(username, password), &token); + } + TestAuth::Digest { username, password } => { + auth.set_supported_methods(gstreamer_rtsp::RTSPAuthMethod::Digest); + auth.add_digest(username, password, &token); + } + } + server.set_auth(Some(&auth)); + } + + server + .mount_points() + .expect("RTSP server has no mount points") + .add_factory("/test", factory); + server.attach(Some(&context)).expect("failed to attach RTSP server"); + let port = server.bound_port(); + assert!(port > 0, "RTSP server reported no bound port"); + log::info!( + "RTSP test server listening at {}://127.0.0.1:{port}/test{}", + if tls { "rtsps" } else { "rtsp" }, + match test_auth { + TestAuth::None => "", + TestAuth::Basic { .. } => " (basic auth)", + TestAuth::Digest { .. } => " (digest auth)", + }, + ); + log::debug!("RTSP test server pipeline: {media_pipeline}"); + + let main_loop = glib::MainLoop::new(Some(&context), false); + let run_loop = main_loop.clone(); + let thread = thread::Builder::new() + .name("rtsp-test-server".to_owned()) + .spawn(move || run_loop.run()) + .expect("failed to spawn RTSP server thread"); + + // A quit before the loop runs would be lost; wait for it to start. + let started = Instant::now(); + while !main_loop.is_running() { + assert!( + started.elapsed() < Duration::from_secs(5), + "RTSP server main loop did not start" + ); + thread::sleep(Duration::from_millis(1)); + } + + Self { main_loop, thread: Some(thread), port, tls } + } + + /// The stream's RTSP or RTSPS URL. + pub fn url(&self) -> String { + let scheme = if self.tls { "rtsps" } else { "rtsp" }; + format!("{scheme}://127.0.0.1:{}/test", self.port) + } +} + +/// Generates a fresh self-signed certificate for the test server. +fn self_signed_certificate() -> gio::TlsCertificate { + let certified = rcgen::generate_simple_self_signed(vec![ + "localhost".to_owned(), + "127.0.0.1".to_owned(), + ]) + .expect("failed to generate a self-signed certificate"); + let pem = format!("{}{}", certified.cert.pem(), certified.key_pair.serialize_pem()); + gio::TlsCertificate::from_pem(&pem).expect("failed to load the certificate into GIO") +} + +impl Drop for RtspTestServer { + fn drop(&mut self) { + self.main_loop.quit(); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +/// A source configuration for `url` with test-friendly timeouts and every +/// optional field unset. +pub fn test_config(url: String) -> RtspVideoSourceConfig { + RtspVideoSourceConfig { + url, + username: None, + password: None, + codec: None, + resolution: None, + connect_timeout_ms: Some(5_000), + idle_timeout_ms: Some(10_000), + accept_invalid_tls_certs: false, + } +} diff --git a/livekit-capture/tests/source_rtsp_test.rs b/livekit-capture/tests/source_rtsp_test.rs new file mode 100644 index 000000000..4692ca7a8 --- /dev/null +++ b/livekit-capture/tests/source_rtsp_test.rs @@ -0,0 +1,416 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Integration tests for the RTSP source against an in-process GStreamer +//! RTSP server. Not run by default; see `tests/README.md`. + +#![cfg(feature = "__test-source-rtsp")] + +mod common; + +// Replaces `#[test]` with a variant that initializes logging, so `log::` +// output from the source and the test server is visible; see tests/README.md. +use test_log::test; + +use common::{ + pull_access_units, + rtsp::{ + default_pipeline, default_pipeline_with_audio, pipeline, test_config, RtspTestServer, + TestCodec, TEST_RESOLUTION, + }, +}; +use livekit_capture::{ + encoded::{h26x::annex_b_nalus, EncodedFrameType, EncodedVideoCodec, EncodedVideoSource}, + primitive::VideoResolution, + sources::rtsp::{RtspVideoSource, RtspVideoSourceConfig}, +}; + +fn h264_nal_types(payload: &[u8]) -> Vec { + annex_b_nalus(payload).iter().map(|nal| nal[0] & 0x1f).collect() +} + +fn h265_nal_types(payload: &[u8]) -> Vec { + annex_b_nalus(payload).iter().map(|nal| (nal[0] >> 1) & 0x3f).collect() +} + +fn assert_increasing_timestamps(access_units: &[livekit_capture::encoded::OwnedEncodedAccessUnit]) { + for window in access_units.windows(2) { + assert!( + window[1].timestamp_us > window[0].timestamp_us, + "timestamps must increase: {} then {}", + window[0].timestamp_us, + window[1].timestamp_us, + ); + } +} + +#[test] +fn streams_h264_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + assert_eq!(source.codec(), EncodedVideoCodec::H264); + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 15); + + // The first access unit must be a self-contained keyframe: subscribers + // can only initialize a decoder from parameter sets inside it. + let first = &access_units[0]; + assert_eq!(first.frame_type, EncodedFrameType::Key); + let nal_types = h264_nal_types(&first.payload); + assert!(nal_types.contains(&7), "keyframe missing SPS: {nal_types:?}"); + assert!(nal_types.contains(&8), "keyframe missing PPS: {nal_types:?}"); + assert!(nal_types.contains(&5), "keyframe missing IDR: {nal_types:?}"); + + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + assert!(access_unit.payload.starts_with(&[0, 0, 0, 1])); + assert_eq!(access_unit.resolution, TEST_RESOLUTION); + } +} + +#[test] +fn discovers_h264_resolution() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + let mut source = + RtspVideoSource::new_blocking(test_config(server.url())).expect("failed to connect"); + + assert_eq!(source.resolution(), TEST_RESOLUTION); + + // The keyframe consumed by discovery must not be lost. + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); + assert_eq!(first.resolution, TEST_RESOLUTION); +} + +#[test] +fn streams_h265_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H265)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H265), + ..test_config(server.url()) + }) + .expect("failed to connect"); + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + + // H.265 keyframes must carry VPS, SPS, and PPS alongside the IDR — via + // the stream or injected from the real server's SDP. + let first = &access_units[0]; + assert_eq!(first.frame_type, EncodedFrameType::Key); + let nal_types = h265_nal_types(&first.payload); + for parameter_set in [32u8, 33, 34] { + assert!(nal_types.contains(¶meter_set), "keyframe missing NAL {parameter_set}"); + } + assert!( + nal_types.iter().any(|nal_type| matches!(nal_type, 19 | 20)), + "keyframe missing IDR: {nal_types:?}" + ); + assert_increasing_timestamps(&access_units); +} + +#[test] +fn streams_vp8_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Vp8)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP8), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // VP8 has no SDP resolution hints; discovery parses the first keyframe. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::VP8); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn rejects_codec_mismatch() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP8), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + assert!(err.to_string().contains("codec mismatch"), "unexpected error: {err}"); +} + +#[test] +fn streams_h264_over_rtsps() { + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H264)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + // The test server's certificate is self-signed. + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect over TLS"); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); +} + +#[test] +fn rejects_untrusted_tls_certificate() { + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H264)); + // Default configuration verifies against the system roots, which must + // reject the server's self-signed certificate. + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + let message = err.to_string(); + assert!( + message.contains("TLS") || message.contains("certificate"), + "unexpected error: {message}" + ); +} + +#[test] +fn authenticates_with_digest_over_rtsps() { + let server = RtspTestServer::launch_tls_with_digest_auth(&default_pipeline(TestCodec::H264), "admin", "secret"); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect with credentials over TLS"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} + +#[test] +fn authenticates_with_digest() { + let server = RtspTestServer::launch_with_digest_auth(&default_pipeline(TestCodec::H264), "admin", "secret"); + + // Without credentials the server's challenge cannot be answered. + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + assert!(err.to_string().contains("credentials"), "unexpected error: {err}"); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + codec: Some(EncodedVideoCodec::H264), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect with credentials"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} + +#[test] +fn selects_video_track_among_audio() { + let server = RtspTestServer::launch(&default_pipeline_with_audio(TestCodec::H264)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // The video track is selected from the two-track SDP; the audio track + // is neither set up nor streamed. + assert_eq!(source.codec(), EncodedVideoCodec::H264); + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::H264); + } +} + +#[test] +fn discovers_cropped_h264_resolution() { + // 1080p is coded as 1088 rows plus SPS frame cropping; discovery must + // report the display resolution from a real encoder's SPS. + let server = RtspTestServer::launch(&pipeline(TestCodec::H264, VideoResolution::new(1920, 1080))); + let source = + RtspVideoSource::new_blocking(test_config(server.url())).expect("failed to connect"); + + assert_eq!(source.resolution(), VideoResolution::new(1920, 1080)); +} + +#[test] +fn streams_vp9_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Vp9)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::VP9), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // Discovery parses the first keyframe's uncompressed header. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::VP9); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn streams_av1_access_units() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::Av1)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::AV1), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + // Discovery parses the sequence header OBU of the first keyframe. + assert_eq!(source.resolution(), TEST_RESOLUTION); + + let access_units = pull_access_units(&mut source, 5); + assert_eq!(access_units[0].frame_type, EncodedFrameType::Key); + assert_increasing_timestamps(&access_units); + for access_unit in &access_units { + assert_eq!(access_unit.codec, EncodedVideoCodec::AV1); + assert!(!access_unit.payload.is_empty()); + } +} + +#[test] +fn rejects_wrong_credentials() { + let server = RtspTestServer::launch_with_digest_auth( + &default_pipeline(TestCodec::H264), + "admin", + "secret", + ); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("wrong".to_owned()), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + + // One authenticated retry, then a clean failure — no retry loop. + assert!(err.to_string().contains("401"), "unexpected error: {err}"); +} + +#[test] +fn rejects_unknown_path() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url().replace("/test", "/wrong")) + }) + .unwrap_err(); + + assert!(err.to_string().contains("404"), "unexpected error: {err}"); +} + +#[test] +fn rejects_mismatched_declared_resolution() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + // A declared resolution skips discovery and is verified against the + // first keyframe instead. + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(VideoResolution::new(1280, 720)), + ..test_config(server.url()) + }) + .expect("failed to connect"); + + let stop = livekit_capture::pump::PumpStop::new(); + let err = source.next_access_unit(&stop).unwrap_err(); + assert!(err.to_string().contains("1280x720"), "unexpected error: {err}"); +} + +#[test] +fn reconnects_after_drop() { + let server = RtspTestServer::launch(&default_pipeline(TestCodec::H264)); + // The previous source's teardown must leave the server usable for a + // fresh connection. + for _ in 0..2 { + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect"); + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); + } +} + +#[test] +fn authenticates_with_basic() { + let server = RtspTestServer::launch_with_basic_auth( + &default_pipeline(TestCodec::H264), + "admin", + "secret", + ); + + let err = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .unwrap_err(); + assert!(err.to_string().contains("credentials"), "unexpected error: {err}"); + + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + username: Some("admin".to_owned()), + password: Some("secret".to_owned()), + resolution: Some(TEST_RESOLUTION), + ..test_config(server.url()) + }) + .expect("failed to connect with credentials"); + + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} + +#[test] +fn streams_h265_over_rtsps() { + let server = RtspTestServer::launch_tls(&default_pipeline(TestCodec::H265)); + let mut source = RtspVideoSource::new_blocking(RtspVideoSourceConfig { + codec: Some(EncodedVideoCodec::H265), + accept_invalid_tls_certs: true, + ..test_config(server.url()) + }) + .expect("failed to connect over TLS"); + + assert_eq!(source.resolution(), TEST_RESOLUTION); + let first = pull_access_units(&mut source, 1).remove(0); + assert_eq!(first.frame_type, EncodedFrameType::Key); +} diff --git a/livekit-ffi-node-bindings/proto/capture_pb.d.ts b/livekit-ffi-node-bindings/proto/capture_pb.d.ts index abba9e80d..d08b41a2e 100644 --- a/livekit-ffi-node-bindings/proto/capture_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/capture_pb.d.ts @@ -201,6 +201,91 @@ export declare class GstreamerVideoSourceConfig extends Message | undefined, b: GstreamerVideoSourceConfig | PlainMessage | undefined): boolean; } +/** + * Encoded ingest from an RTSP server over TCP-interleaved RTP. + * + * @generated from message livekit.proto.RtspVideoSourceConfig + */ +export declare class RtspVideoSourceConfig extends Message { + /** + * RTSP URL (rtsp://host[:port]/path, or rtsps:// when the server is built + * with TLS support). URL userinfo is accepted and stripped from requests; + * `username`/`password` take precedence over it. + * + * @generated from field: required string url = 1; + */ + url?: string; + + /** + * Username for RTSP authentication, overriding URL userinfo. + * + * @generated from field: optional string username = 2; + */ + username?: string; + + /** + * Password for RTSP authentication, overriding URL userinfo. + * + * @generated from field: optional string password = 3; + */ + password?: string; + + /** + * Codec required from the stream; the first supported video track offered + * by the SDP is used when omitted. + * + * @generated from field: optional livekit.proto.VideoCodec codec = 4; + */ + codec?: VideoCodec; + + /** + * Encoded frame resolution. When omitted, it is discovered from the SDP + * when declared there, and from the stream's first keyframe otherwise; + * when set, the first keyframe is verified against it. + * + * @generated from field: optional livekit.proto.VideoSourceResolution resolution = 5; + */ + resolution?: VideoSourceResolution; + + /** + * TCP connect and RTSP handshake timeout in milliseconds (default 10000). + * + * @generated from field: optional uint32 connect_timeout_ms = 6; + */ + connectTimeoutMs?: number; + + /** + * Maximum tolerated stream silence in milliseconds (default 30000). + * + * @generated from field: optional uint32 idle_timeout_ms = 7; + */ + idleTimeoutMs?: number; + + /** + * Disables TLS certificate verification for rtsps:// URLs. Most cameras + * present self-signed certificates, which fail verification against the + * system roots. The connection stays encrypted but is not authenticated: + * a network attacker could impersonate the camera. + * + * @generated from field: optional bool accept_invalid_tls_certs = 8; + */ + acceptInvalidTlsCerts?: boolean; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.RtspVideoSourceConfig"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): RtspVideoSourceConfig; + + static fromJson(jsonValue: JsonValue, options?: Partial): RtspVideoSourceConfig; + + static fromJsonString(jsonString: string, options?: Partial): RtspVideoSourceConfig; + + static equals(a: RtspVideoSourceConfig | PlainMessage | undefined, b: RtspVideoSourceConfig | PlainMessage | undefined): boolean; +} + /** * Test pattern rendered on the GPU. * @@ -394,6 +479,12 @@ export declare class NewCaptureSourceRequest extends Message [ + { no: 1, name: "url", kind: "scalar", T: 9 /* ScalarType.STRING */, req: true }, + { no: 2, name: "username", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 3, name: "password", kind: "scalar", T: 9 /* ScalarType.STRING */, opt: true }, + { no: 4, name: "codec", kind: "enum", T: proto2.getEnumType(VideoCodec), opt: true }, + { no: 5, name: "resolution", kind: "message", T: VideoSourceResolution, opt: true }, + { no: 6, name: "connect_timeout_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true }, + { no: 7, name: "idle_timeout_ms", kind: "scalar", T: 13 /* ScalarType.UINT32 */, opt: true }, + { no: 8, name: "accept_invalid_tls_certs", kind: "scalar", T: 8 /* ScalarType.BOOL */, opt: true }, + ], +); + /** * Test pattern rendered on the GPU. * @@ -174,6 +193,7 @@ const NewCaptureSourceRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 1, name: "gstreamer", kind: "message", T: GstreamerVideoSourceConfig, oneof: "config" }, { no: 2, name: "pattern", kind: "message", T: PatternVideoSourceConfig, oneof: "config" }, { no: 5, name: "clock", kind: "message", T: ClockVideoSourceConfig, oneof: "config" }, + { no: 6, name: "rtsp", kind: "message", T: RtspVideoSourceConfig, oneof: "config" }, { no: 3, name: "request_async_id", kind: "scalar", T: 4 /* ScalarType.UINT64 */, opt: true }, ], ); @@ -289,6 +309,7 @@ exports.CaptureSourceKind = CaptureSourceKind; exports.CaptureExit = CaptureExit; exports.GstreamerRateControl = GstreamerRateControl; exports.GstreamerVideoSourceConfig = GstreamerVideoSourceConfig; +exports.RtspVideoSourceConfig = RtspVideoSourceConfig; exports.PatternVideoSourceConfig = PatternVideoSourceConfig; exports.ClockVideoSourceConfig = ClockVideoSourceConfig; exports.CaptureSourceInfo = CaptureSourceInfo; diff --git a/livekit-ffi/Cargo.toml b/livekit-ffi/Cargo.toml index 1279518f7..1ae45c283 100644 --- a/livekit-ffi/Cargo.toml +++ b/livekit-ffi/Cargo.toml @@ -26,6 +26,8 @@ capture = ["dep:livekit-capture"] capture-clock = ["capture", "livekit-capture/source-clock"] capture-gstreamer = ["capture", "livekit-capture/source-gstreamer"] capture-pattern = ["capture", "livekit-capture/source-pattern"] +capture-rtsp = ["capture", "livekit-capture/source-rtsp"] +capture-rtsp-tls = ["capture-rtsp", "livekit-capture/source-rtsp-tls"] [dependencies] livekit = { workspace = true } diff --git a/livekit-ffi/protocol/capture.proto b/livekit-ffi/protocol/capture.proto index 331d62dfa..b4ddadc68 100644 --- a/livekit-ffi/protocol/capture.proto +++ b/livekit-ffi/protocol/capture.proto @@ -72,6 +72,34 @@ message GstreamerVideoSourceConfig { optional GstreamerRateControl rate_control = 4; } +// Encoded ingest from an RTSP server over TCP-interleaved RTP. +message RtspVideoSourceConfig { + // RTSP URL (rtsp://host[:port]/path, or rtsps:// when the server is built + // with TLS support). URL userinfo is accepted and stripped from requests; + // `username`/`password` take precedence over it. + required string url = 1; + // Username for RTSP authentication, overriding URL userinfo. + optional string username = 2; + // Password for RTSP authentication, overriding URL userinfo. + optional string password = 3; + // Codec required from the stream; the first supported video track offered + // by the SDP is used when omitted. + optional VideoCodec codec = 4; + // Encoded frame resolution. When omitted, it is discovered from the SDP + // when declared there, and from the stream's first keyframe otherwise; + // when set, the first keyframe is verified against it. + optional VideoSourceResolution resolution = 5; + // TCP connect and RTSP handshake timeout in milliseconds (default 10000). + optional uint32 connect_timeout_ms = 6; + // Maximum tolerated stream silence in milliseconds (default 30000). + optional uint32 idle_timeout_ms = 7; + // Disables TLS certificate verification for rtsps:// URLs. Most cameras + // present self-signed certificates, which fail verification against the + // system roots. The connection stays encrypted but is not authenticated: + // a network attacker could impersonate the camera. + optional bool accept_invalid_tls_certs = 8; +} + // Test patterns built into livekit-capture. enum Pattern { // Animated color gradient. @@ -137,6 +165,7 @@ message NewCaptureSourceRequest { GstreamerVideoSourceConfig gstreamer = 1; PatternVideoSourceConfig pattern = 2; ClockVideoSourceConfig clock = 5; + RtspVideoSourceConfig rtsp = 6; } optional uint64 request_async_id = 3; } diff --git a/livekit-ffi/src/conversion/capture.rs b/livekit-ffi/src/conversion/capture.rs index 372d12754..5bcb3071e 100644 --- a/livekit-ffi/src/conversion/capture.rs +++ b/livekit-ffi/src/conversion/capture.rs @@ -15,7 +15,11 @@ use crate::proto; use livekit_capture::{encoded::EncodedVideoCodec, primitive::VideoResolution}; -#[cfg(any(feature = "capture-gstreamer", feature = "capture-pattern"))] +#[cfg(any( + feature = "capture-gstreamer", + feature = "capture-pattern", + feature = "capture-rtsp" +))] use crate::{FfiError, FfiResult}; #[cfg(feature = "capture-clock")] use livekit_capture::sources::clock::ClockVideoSourceConfig; @@ -25,6 +29,8 @@ use livekit_capture::sources::gstreamer::{ }; #[cfg(feature = "capture-pattern")] use livekit_capture::sources::pattern::{Pattern, PatternVideoSourceConfig}; +#[cfg(feature = "capture-rtsp")] +use livekit_capture::sources::rtsp::RtspVideoSourceConfig; impl From for VideoResolution { fn from(resolution: proto::VideoSourceResolution) -> Self { @@ -65,7 +71,7 @@ impl From for GStreamerBitrateUnit { } } -#[cfg(feature = "capture-gstreamer")] +#[cfg(any(feature = "capture-gstreamer", feature = "capture-rtsp"))] pub fn video_codec_from_proto(codec: proto::VideoCodec) -> EncodedVideoCodec { match codec { proto::VideoCodec::H264 => EncodedVideoCodec::H264, @@ -89,6 +95,31 @@ pub fn video_codec_to_proto(codec: EncodedVideoCodec) -> Option FfiResult { + let codec = config + .codec + .map(|value| { + proto::VideoCodec::try_from(value) + .map(video_codec_from_proto) + .map_err(|_| FfiError::InvalidRequest("invalid codec".into())) + }) + .transpose()?; + + Ok(RtspVideoSourceConfig { + url: config.url, + username: config.username, + password: config.password, + codec, + resolution: config.resolution.map(VideoResolution::from), + connect_timeout_ms: config.connect_timeout_ms, + idle_timeout_ms: config.idle_timeout_ms, + accept_invalid_tls_certs: config.accept_invalid_tls_certs.unwrap_or_default(), + }) +} + #[cfg(feature = "capture-gstreamer")] pub fn gstreamer_config_from_proto( config: proto::GstreamerVideoSourceConfig, diff --git a/livekit-ffi/src/server/capture.rs b/livekit-ffi/src/server/capture.rs index e5e202ebd..abb74aacc 100644 --- a/livekit-ffi/src/server/capture.rs +++ b/livekit-ffi/src/server/capture.rs @@ -32,12 +32,16 @@ use livekit_capture::sources::clock::ClockVideoSource; use livekit_capture::sources::gstreamer::GStreamerVideoSource; #[cfg(feature = "capture-pattern")] use livekit_capture::sources::pattern::PatternVideoSource; +#[cfg(feature = "capture-rtsp")] +use livekit_capture::sources::rtsp::RtspVideoSource; use super::{video_source::FfiVideoSource, FfiHandle, FfiServer}; #[cfg(feature = "capture-gstreamer")] use crate::conversion::capture::gstreamer_config_from_proto; #[cfg(feature = "capture-pattern")] use crate::conversion::capture::pattern_config_from_proto; +#[cfg(feature = "capture-rtsp")] +use crate::conversion::capture::rtsp_config_from_proto; use crate::{conversion::capture::video_codec_to_proto, proto, FfiError, FfiHandleId, FfiResult}; /// A capture pump of either kind, boxed at the FFI edge. @@ -120,6 +124,14 @@ async fn create_capture_source( let source: Box = Box::new(source); CapturePump::Encoded(EncodedVideoPump::new(source)) } + #[cfg(feature = "capture-rtsp")] + proto::new_capture_source_request::Config::Rtsp(config) => { + let source = RtspVideoSource::new(rtsp_config_from_proto(config)?) + .await + .map_err(|err| FfiError::InvalidRequest(err.to_string().into()))?; + let source: Box = Box::new(source); + CapturePump::Encoded(EncodedVideoPump::new(source)) + } #[cfg(feature = "capture-pattern")] proto::new_capture_source_request::Config::Pattern(config) => { let source = PatternVideoSource::new(pattern_config_from_proto(config)?)