diff --git a/.changeset/expose_pre_encoded_video_to_ffi.md b/.changeset/expose_pre_encoded_video_to_ffi.md new file mode 100644 index 000000000..5b09fb0aa --- /dev/null +++ b/.changeset/expose_pre_encoded_video_to_ffi.md @@ -0,0 +1,5 @@ +--- +livekit-ffi: minor +--- + +Exposes the pre-encoded video publish path to ffi \ No newline at end of file diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 53ea782bd..a8ee2c63a 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -87,6 +87,7 @@ message FfiRequest { NewVideoStreamRequest new_video_stream = 20; NewVideoSourceRequest new_video_source = 21; CaptureVideoFrameRequest capture_video_frame = 22; + CaptureEncodedVideoFrameRequest capture_encoded_video_frame = 87; VideoConvertRequest video_convert = 23; VideoStreamFromParticipantRequest video_stream_from_participant = 24; @@ -184,7 +185,7 @@ message FfiRequest { // Room event ready signal ReadyForRoomEventRequest ready_for_room_event = 83; - // NEXT_ID: 87 + // NEXT_ID: 88 } } @@ -219,6 +220,7 @@ message FfiResponse { NewVideoStreamResponse new_video_stream = 20; NewVideoSourceResponse new_video_source = 21; CaptureVideoFrameResponse capture_video_frame = 22; + CaptureEncodedVideoFrameResponse capture_encoded_video_frame = 87; VideoConvertResponse video_convert = 23; VideoStreamFromParticipantResponse video_stream_from_participant = 24; @@ -313,7 +315,7 @@ message FfiResponse { // Room event ready signal ReadyForRoomEventResponse ready_for_room_event = 82; - // NEXT_ID: 87 + // NEXT_ID: 88 } } @@ -381,8 +383,9 @@ message FfiEvent { // Data Track (schemas) DefineSchemaCallback define_schema = 45; GetSchemaCallback get_schema = 46; + VideoSourceEvent video_source_event = 47; - // NEXT_ID: 47 + // NEXT_ID: 48 } } diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index a4b610e0e..8463fac68 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -68,6 +68,9 @@ message NewVideoSourceRequest { // Most of the time it corresponds to the source resolution required VideoSourceResolution resolution = 2; optional bool is_screencast = 3; + // When true, creates a source for pre-encoded access units via + // CaptureEncodedVideoFrameRequest instead of raw CaptureVideoFrameRequest. + optional bool encoded = 4; } message NewVideoSourceResponse { required OwnedVideoSource source = 1; } @@ -82,6 +85,53 @@ message CaptureVideoFrameRequest { message CaptureVideoFrameResponse {} +// Codec of a pre-encoded video access unit +enum EncodedVideoCodec { + ENCODED_CODEC_H264 = 0; + ENCODED_CODEC_H265 = 1; + ENCODED_CODEC_VP8 = 2; + ENCODED_CODEC_VP9 = 3; + ENCODED_CODEC_AV1 = 4; +} + +// Frame type of a pre-encoded video access unit +enum EncodedFrameType { + ENCODED_FRAME_KEY = 0; + ENCODED_FRAME_DELTA = 1; +} + +// Push a pre-encoded access unit to an encoded VideoSource +message CaptureEncodedVideoFrameRequest { + required uint64 source_handle = 1; + required EncodedVideoCodec codec = 2; + required bytes data = 3; + required EncodedFrameType frame_type = 4; + required uint32 width = 5; + required uint32 height = 6; + required int64 timestamp_us = 7; // In microseconds + optional FrameMetadata metadata = 8; +} + +message CaptureEncodedVideoFrameResponse {} + +// Encoder rate-control feedback from WebRTC back to the FFI client. +// Pushed periodically for an encoded source when WebRTC updates the +// bitrate/framerate target, and whenever a keyframe is requested. +message EncodedRateControlEvent { + required uint64 source_handle = 1; + optional uint64 target_bitrate_bps = 2; + optional double framerate_fps = 3; + required bool keyframe_requested = 4; +} + +// Wraps all async events originating from a VideoSource +message VideoSourceEvent { + required uint64 source_handle = 1; + oneof message { + EncodedRateControlEvent rate_control = 2; + } +} + message VideoConvertRequest { optional bool flip_y = 1; required VideoBufferInfo buffer = 2; diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8ee569885..0e9a22f45 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -520,6 +520,16 @@ unsafe fn on_capture_video_frame( Ok(proto::CaptureVideoFrameResponse::default()) } +/// Push a pre-encoded access unit to a source created with `encoded = true` +unsafe fn on_capture_encoded_video_frame( + server: &'static FfiServer, + push: proto::CaptureEncodedVideoFrameRequest, +) -> FfiResult { + let source = server.retrieve_handle::(push.source_handle)?; + source.capture_encoded_frame(server, push)?; + Ok(proto::CaptureEncodedVideoFrameResponse::default()) +} + /// Convert a video frame /// /// # Safety: The user must ensure that the pointers/len provided are valid @@ -1390,6 +1400,9 @@ pub fn handle_request( } Request::NewVideoSource(req) => on_new_video_source(server, req)?.into(), Request::CaptureVideoFrame(req) => unsafe { on_capture_video_frame(server, req)?.into() }, + Request::CaptureEncodedVideoFrame(req) => unsafe { + on_capture_encoded_video_frame(server, req)?.into() + }, Request::VideoConvert(req) => unsafe { on_video_convert(server, req)?.into() }, Request::NewAudioStream(req) => on_new_audio_stream(server, req)?.into(), Request::NewAudioSource(req) => on_new_audio_source(server, req)?.into(), diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index ee1410a72..0706513ca 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -53,15 +53,54 @@ impl FfiVideoSource { proto::VideoSourceType::VideoSourceNative => { use livekit::webrtc::video_source::native::NativeVideoSource; - let is_screencast = new_source.is_screencast.unwrap_or(false); - let video_source = - NativeVideoSource::new(new_source.resolution.into(), is_screencast); + let video_source = if new_source.encoded.unwrap_or(false) { + NativeVideoSource::new_encoded(new_source.resolution.into()) + } else { + let is_screencast = new_source.is_screencast.unwrap_or(false); + NativeVideoSource::new(new_source.resolution.into(), is_screencast) + }; RtcVideoSource::Native(video_source) } _ => return Err(FfiError::InvalidRequest("unsupported video source type".into())), }; let handle_id = server.next_id(); + + #[cfg(not(target_arch = "wasm32"))] + if let RtcVideoSource::Native(ref native_source) = source_inner { + if new_source.encoded.unwrap_or(false) { + let native_source = native_source.clone(); + let handle = server.async_runtime.spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_millis(100)); + loop { + interval.tick().await; + + let keyframe_requested = native_source.take_keyframe_request(); + let rate_control = native_source.take_rate_control_request(); + + if keyframe_requested || rate_control.is_some() { + let event = proto::EncodedRateControlEvent { + source_handle: handle_id, + target_bitrate_bps: rate_control.map(|r| r.target_bitrate_bps), + framerate_fps: rate_control.map(|r| r.framerate_fps), + keyframe_requested, + }; + let _ = server.send_event( + proto::VideoSourceEvent { + source_handle: handle_id, + message: Some(proto::video_source_event::Message::RateControl( + event, + )), + } + .into(), + ); + } + } + }); + server.watch_panic(handle); + } + } + let video_source = Self { handle_id, source_type, source: source_inner }; let source_info = proto::VideoSourceInfo::from(&video_source); server.store_handle(handle_id, video_source); @@ -94,6 +133,47 @@ impl FfiVideoSource { } Ok(()) } + + pub unsafe fn capture_encoded_frame( + &self, + _server: &'static server::FfiServer, + capture: proto::CaptureEncodedVideoFrameRequest, + ) -> FfiResult<()> { + match self.source { + #[cfg(not(target_arch = "wasm32"))] + RtcVideoSource::Native(ref source) => { + use livekit::webrtc::video_frame::{ + EncodedFrameType, EncodedVideoCodec, EncodedVideoFrame, + }; + use livekit::webrtc::video_source::VideoResolution; + + let codec = match capture.codec() { + proto::EncodedVideoCodec::EncodedCodecH264 => EncodedVideoCodec::H264, + proto::EncodedVideoCodec::EncodedCodecH265 => EncodedVideoCodec::H265, + proto::EncodedVideoCodec::EncodedCodecVp8 => EncodedVideoCodec::VP8, + proto::EncodedVideoCodec::EncodedCodecVp9 => EncodedVideoCodec::VP9, + proto::EncodedVideoCodec::EncodedCodecAv1 => EncodedVideoCodec::AV1, + }; + let frame_type = match capture.frame_type() { + proto::EncodedFrameType::EncodedFrameKey => EncodedFrameType::Key, + proto::EncodedFrameType::EncodedFrameDelta => EncodedFrameType::Delta, + }; + + let frame = EncodedVideoFrame { + codec, + payload: &capture.data, + timestamp_us: capture.timestamp_us, + frame_type, + resolution: VideoResolution { width: capture.width, height: capture.height }, + frame_metadata: frame_metadata_from_proto(capture.metadata), + }; + + source.capture_encoded_frame(&frame); + } + _ => {} + } + Ok(()) + } } #[cfg(test)]