diff --git a/.changeset/pre-encoded-ingest.md b/.changeset/pre-encoded-ingest.md new file mode 100644 index 000000000..23220a75b --- /dev/null +++ b/.changeset/pre-encoded-ingest.md @@ -0,0 +1,5 @@ +--- +livekit-ffi: patch +--- + +Add FFI support for publishing pre-encoded video frames and consuming encoder feedback. diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts index ac81cb93e..974aa426a 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/ffi_pb.d.ts @@ -21,7 +21,7 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialM import { Message, proto2 } from "@bufbuild/protobuf"; import type { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, ReadyForRoomEventRequest, ReadyForRoomEventResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, SimulateScenarioCallback, SimulateScenarioRequest, SimulateScenarioResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js"; import type { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } from "./track_pb.js"; -import type { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js"; +import type { CaptureEncodedVideoFrameRequest, CaptureEncodedVideoFrameResponse, CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, TakeEncodedVideoSourceFeedbackRequest, TakeEncodedVideoSourceFeedbackResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js"; import type { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse, StartRecordingRequest, StartRecordingResponse, StopRecordingRequest, StopRecordingResponse } from "./audio_frame_pb.js"; import type { E2eeRequest, E2eeResponse } from "./e2ee_pb.js"; import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } from "./rpc_pb.js"; @@ -557,6 +557,20 @@ export declare class FfiRequest extends Message { */ value: GetSchemaRequest; case: "getSchema"; + } | { + /** + * Pre-encoded video + * + * @generated from field: livekit.proto.CaptureEncodedVideoFrameRequest capture_encoded_video_frame = 87; + */ + value: CaptureEncodedVideoFrameRequest; + case: "captureEncodedVideoFrame"; + } | { + /** + * @generated from field: livekit.proto.TakeEncodedVideoSourceFeedbackRequest take_encoded_video_source_feedback = 88; + */ + value: TakeEncodedVideoSourceFeedbackRequest; + case: "takeEncodedVideoSourceFeedback"; } | { /** * Reconnection / chaos testing @@ -1119,6 +1133,20 @@ export declare class FfiResponse extends Message { */ value: GetSchemaResponse; case: "getSchema"; + } | { + /** + * Pre-encoded video + * + * @generated from field: livekit.proto.CaptureEncodedVideoFrameResponse capture_encoded_video_frame = 87; + */ + value: CaptureEncodedVideoFrameResponse; + case: "captureEncodedVideoFrame"; + } | { + /** + * @generated from field: livekit.proto.TakeEncodedVideoSourceFeedbackResponse take_encoded_video_source_feedback = 88; + */ + value: TakeEncodedVideoSourceFeedbackResponse; + case: "takeEncodedVideoSourceFeedback"; } | { /** * Reconnection / chaos testing diff --git a/livekit-ffi-node-bindings/proto/ffi_pb.js b/livekit-ffi-node-bindings/proto/ffi_pb.js index 4f0bdd422..6fcfc95a3 100644 --- a/livekit-ffi-node-bindings/proto/ffi_pb.js +++ b/livekit-ffi-node-bindings/proto/ffi_pb.js @@ -23,7 +23,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); const { proto2 } = require("@bufbuild/protobuf"); const { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, ReadyForRoomEventRequest, ReadyForRoomEventResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, SimulateScenarioCallback, SimulateScenarioRequest, SimulateScenarioResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } = require("./room_pb.js"); const { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } = require("./track_pb.js"); -const { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } = require("./video_frame_pb.js"); +const { CaptureEncodedVideoFrameRequest, CaptureEncodedVideoFrameResponse, CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, TakeEncodedVideoSourceFeedbackRequest, TakeEncodedVideoSourceFeedbackResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } = require("./video_frame_pb.js"); const { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse, StartRecordingRequest, StartRecordingResponse, StopRecordingRequest, StopRecordingResponse } = require("./audio_frame_pb.js"); const { E2eeRequest, E2eeResponse } = require("./e2ee_pb.js"); const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } = require("./rpc_pb.js"); @@ -131,6 +131,8 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType( { no: 84, name: "remote_data_track_set_pipeline_options", kind: "message", T: RemoteDataTrackSetPipelineOptionsRequest, oneof: "message" }, { no: 85, name: "define_schema", kind: "message", T: DefineSchemaRequest, oneof: "message" }, { no: 86, name: "get_schema", kind: "message", T: GetSchemaRequest, oneof: "message" }, + { no: 87, name: "capture_encoded_video_frame", kind: "message", T: CaptureEncodedVideoFrameRequest, oneof: "message" }, + { no: 88, name: "take_encoded_video_source_feedback", kind: "message", T: TakeEncodedVideoSourceFeedbackRequest, oneof: "message" }, { no: 76, name: "simulate_scenario", kind: "message", T: SimulateScenarioRequest, oneof: "message" }, { no: 77, name: "new_platform_audio", kind: "message", T: NewPlatformAudioRequest, oneof: "message" }, { no: 78, name: "get_audio_devices", kind: "message", T: GetAudioDevicesRequest, oneof: "message" }, @@ -226,6 +228,8 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType( { no: 84, name: "remote_data_track_set_pipeline_options", kind: "message", T: RemoteDataTrackSetPipelineOptionsResponse, oneof: "message" }, { no: 85, name: "define_schema", kind: "message", T: DefineSchemaResponse, oneof: "message" }, { no: 86, name: "get_schema", kind: "message", T: GetSchemaResponse, oneof: "message" }, + { no: 87, name: "capture_encoded_video_frame", kind: "message", T: CaptureEncodedVideoFrameResponse, oneof: "message" }, + { no: 88, name: "take_encoded_video_source_feedback", kind: "message", T: TakeEncodedVideoSourceFeedbackResponse, oneof: "message" }, { no: 75, name: "simulate_scenario", kind: "message", T: SimulateScenarioResponse, oneof: "message" }, { no: 76, name: "new_platform_audio", kind: "message", T: NewPlatformAudioResponse, oneof: "message" }, { no: 77, name: "get_audio_devices", kind: "message", T: GetAudioDevicesResponse, oneof: "message" }, diff --git a/livekit-ffi-node-bindings/proto/room_pb.d.ts b/livekit-ffi-node-bindings/proto/room_pb.d.ts index 069ca8f5c..a39438533 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/room_pb.d.ts @@ -125,6 +125,11 @@ export declare enum VideoEncoderBackend { * @generated from enum value: ENCODER_BACKEND_VIDEOTOOLBOX = 5; */ ENCODER_BACKEND_VIDEOTOOLBOX = 5, + + /** + * @generated from enum value: ENCODER_BACKEND_PRE_ENCODED = 6; + */ + ENCODER_BACKEND_PRE_ENCODED = 6, } /** diff --git a/livekit-ffi-node-bindings/proto/room_pb.js b/livekit-ffi-node-bindings/proto/room_pb.js index 6a6a5ed44..539db79d9 100644 --- a/livekit-ffi-node-bindings/proto/room_pb.js +++ b/livekit-ffi-node-bindings/proto/room_pb.js @@ -65,6 +65,7 @@ const VideoEncoderBackend = /*@__PURE__*/ proto2.makeEnum( {no: 3, name: "ENCODER_BACKEND_NVENC"}, {no: 4, name: "ENCODER_BACKEND_VAAPI"}, {no: 5, name: "ENCODER_BACKEND_VIDEOTOOLBOX"}, + {no: 6, name: "ENCODER_BACKEND_PRE_ENCODED"}, ], ); diff --git a/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts b/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts index 76a56bb4d..8b9f9f96a 100644 --- a/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts +++ b/livekit-ffi-node-bindings/proto/video_frame_pb.d.ts @@ -52,6 +52,21 @@ export declare enum VideoCodec { H265 = 4, } +/** + * @generated from enum livekit.proto.EncodedFrameType + */ +export declare enum EncodedFrameType { + /** + * @generated from enum value: ENCODED_FRAME_KEY = 0; + */ + ENCODED_FRAME_KEY = 0, + + /** + * @generated from enum value: ENCODED_FRAME_DELTA = 1; + */ + ENCODED_FRAME_DELTA = 1, +} + /** * @generated from enum livekit.proto.VideoRotation */ @@ -168,6 +183,11 @@ export declare enum VideoSourceType { * @generated from enum value: VIDEO_SOURCE_NATIVE = 0; */ VIDEO_SOURCE_NATIVE = 0, + + /** + * @generated from enum value: VIDEO_SOURCE_ENCODED = 1; + */ + VIDEO_SOURCE_ENCODED = 1, } /** @@ -469,6 +489,210 @@ export declare class CaptureVideoFrameResponse extends Message | undefined, b: CaptureVideoFrameResponse | PlainMessage | undefined): boolean; } +/** + * Foreign-memory buffer containing one encoded video access unit. + * The payload pointer only needs to remain valid for the duration of the + * synchronous FFI request; the native implementation copies it before + * returning. + * + * @generated from message livekit.proto.EncodedVideoBufferInfo + */ +export declare class EncodedVideoBufferInfo extends Message { + /** + * @generated from field: required uint64 data_ptr = 1; + */ + dataPtr?: bigint; + + /** + * @generated from field: required uint64 data_len = 2; + */ + dataLen?: bigint; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.EncodedVideoBufferInfo"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): EncodedVideoBufferInfo; + + static fromJson(jsonValue: JsonValue, options?: Partial): EncodedVideoBufferInfo; + + static fromJsonString(jsonString: string, options?: Partial): EncodedVideoBufferInfo; + + static equals(a: EncodedVideoBufferInfo | PlainMessage | undefined, b: EncodedVideoBufferInfo | PlainMessage | undefined): boolean; +} + +/** + * Push one complete pre-encoded access unit to an encoded VideoSource. + * + * @generated from message livekit.proto.CaptureEncodedVideoFrameRequest + */ +export declare class CaptureEncodedVideoFrameRequest extends Message { + /** + * @generated from field: required uint64 source_handle = 1; + */ + sourceHandle?: bigint; + + /** + * @generated from field: required livekit.proto.EncodedVideoBufferInfo buffer = 2; + */ + buffer?: EncodedVideoBufferInfo; + + /** + * @generated from field: required livekit.proto.VideoCodec codec = 3; + */ + codec?: VideoCodec; + + /** + * @generated from field: required livekit.proto.EncodedFrameType frame_type = 4; + */ + frameType?: EncodedFrameType; + + /** + * @generated from field: required uint32 width = 5; + */ + width?: number; + + /** + * @generated from field: required uint32 height = 6; + */ + height?: number; + + /** + * @generated from field: required int64 timestamp_us = 7; + */ + timestampUs?: bigint; + + /** + * @generated from field: optional livekit.proto.FrameMetadata metadata = 8; + */ + metadata?: FrameMetadata; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.CaptureEncodedVideoFrameRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): CaptureEncodedVideoFrameRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): CaptureEncodedVideoFrameRequest; + + static fromJsonString(jsonString: string, options?: Partial): CaptureEncodedVideoFrameRequest; + + static equals(a: CaptureEncodedVideoFrameRequest | PlainMessage | undefined, b: CaptureEncodedVideoFrameRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.CaptureEncodedVideoFrameResponse + */ +export declare class CaptureEncodedVideoFrameResponse extends Message { + /** + * @generated from field: required bool accepted = 1; + */ + accepted?: boolean; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.CaptureEncodedVideoFrameResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): CaptureEncodedVideoFrameResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): CaptureEncodedVideoFrameResponse; + + static fromJsonString(jsonString: string, options?: Partial): CaptureEncodedVideoFrameResponse; + + static equals(a: CaptureEncodedVideoFrameResponse | PlainMessage | undefined, b: CaptureEncodedVideoFrameResponse | PlainMessage | undefined): boolean; +} + +/** + * Consumes feedback accumulated by the pre-encoded passthrough encoder since + * the previous call. + * + * @generated from message livekit.proto.TakeEncodedVideoSourceFeedbackRequest + */ +export declare class TakeEncodedVideoSourceFeedbackRequest extends Message { + /** + * @generated from field: required uint64 source_handle = 1; + */ + sourceHandle?: bigint; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.TakeEncodedVideoSourceFeedbackRequest"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): TakeEncodedVideoSourceFeedbackRequest; + + static fromJson(jsonValue: JsonValue, options?: Partial): TakeEncodedVideoSourceFeedbackRequest; + + static fromJsonString(jsonString: string, options?: Partial): TakeEncodedVideoSourceFeedbackRequest; + + static equals(a: TakeEncodedVideoSourceFeedbackRequest | PlainMessage | undefined, b: TakeEncodedVideoSourceFeedbackRequest | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.EncodedRateControl + */ +export declare class EncodedRateControl extends Message { + /** + * @generated from field: required uint64 target_bitrate_bps = 1; + */ + targetBitrateBps?: bigint; + + /** + * @generated from field: required double framerate_fps = 2; + */ + framerateFps?: number; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.EncodedRateControl"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): EncodedRateControl; + + static fromJson(jsonValue: JsonValue, options?: Partial): EncodedRateControl; + + static fromJsonString(jsonString: string, options?: Partial): EncodedRateControl; + + static equals(a: EncodedRateControl | PlainMessage | undefined, b: EncodedRateControl | PlainMessage | undefined): boolean; +} + +/** + * @generated from message livekit.proto.TakeEncodedVideoSourceFeedbackResponse + */ +export declare class TakeEncodedVideoSourceFeedbackResponse extends Message { + /** + * @generated from field: required bool keyframe_requested = 1; + */ + keyframeRequested?: boolean; + + /** + * @generated from field: optional livekit.proto.EncodedRateControl rate_control = 2; + */ + rateControl?: EncodedRateControl; + + constructor(data?: PartialMessage); + + static readonly runtime: typeof proto2; + static readonly typeName = "livekit.proto.TakeEncodedVideoSourceFeedbackResponse"; + static readonly fields: FieldList; + + static fromBinary(bytes: Uint8Array, options?: Partial): TakeEncodedVideoSourceFeedbackResponse; + + static fromJson(jsonValue: JsonValue, options?: Partial): TakeEncodedVideoSourceFeedbackResponse; + + static fromJsonString(jsonString: string, options?: Partial): TakeEncodedVideoSourceFeedbackResponse; + + static equals(a: TakeEncodedVideoSourceFeedbackResponse | PlainMessage | undefined, b: TakeEncodedVideoSourceFeedbackResponse | PlainMessage | undefined): boolean; +} + /** * @generated from message livekit.proto.VideoConvertRequest */ diff --git a/livekit-ffi-node-bindings/proto/video_frame_pb.js b/livekit-ffi-node-bindings/proto/video_frame_pb.js index 8cffe1708..d10134ea7 100644 --- a/livekit-ffi-node-bindings/proto/video_frame_pb.js +++ b/livekit-ffi-node-bindings/proto/video_frame_pb.js @@ -38,6 +38,17 @@ const VideoCodec = /*@__PURE__*/ proto2.makeEnum( ], ); +/** + * @generated from enum livekit.proto.EncodedFrameType + */ +const EncodedFrameType = /*@__PURE__*/ proto2.makeEnum( + "livekit.proto.EncodedFrameType", + [ + {no: 0, name: "ENCODED_FRAME_KEY"}, + {no: 1, name: "ENCODED_FRAME_DELTA"}, + ], +); + /** * @generated from enum livekit.proto.VideoRotation */ @@ -93,6 +104,7 @@ const VideoSourceType = /*@__PURE__*/ proto2.makeEnum( "livekit.proto.VideoSourceType", [ {no: 0, name: "VIDEO_SOURCE_NATIVE"}, + {no: 1, name: "VIDEO_SOURCE_ENCODED"}, ], ); @@ -199,6 +211,86 @@ const CaptureVideoFrameResponse = /*@__PURE__*/ proto2.makeMessageType( [], ); +/** + * Foreign-memory buffer containing one encoded video access unit. + * The payload pointer only needs to remain valid for the duration of the + * synchronous FFI request; the native implementation copies it before + * returning. + * + * @generated from message livekit.proto.EncodedVideoBufferInfo + */ +const EncodedVideoBufferInfo = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.EncodedVideoBufferInfo", + () => [ + { no: 1, name: "data_ptr", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + { no: 2, name: "data_len", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + ], +); + +/** + * Push one complete pre-encoded access unit to an encoded VideoSource. + * + * @generated from message livekit.proto.CaptureEncodedVideoFrameRequest + */ +const CaptureEncodedVideoFrameRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.CaptureEncodedVideoFrameRequest", + () => [ + { no: 1, name: "source_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + { no: 2, name: "buffer", kind: "message", T: EncodedVideoBufferInfo, req: true }, + { no: 3, name: "codec", kind: "enum", T: proto2.getEnumType(VideoCodec), req: true }, + { no: 4, name: "frame_type", kind: "enum", T: proto2.getEnumType(EncodedFrameType), req: true }, + { no: 5, name: "width", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, + { no: 6, name: "height", kind: "scalar", T: 13 /* ScalarType.UINT32 */, req: true }, + { no: 7, name: "timestamp_us", kind: "scalar", T: 3 /* ScalarType.INT64 */, req: true }, + { no: 8, name: "metadata", kind: "message", T: FrameMetadata, opt: true }, + ], +); + +/** + * @generated from message livekit.proto.CaptureEncodedVideoFrameResponse + */ +const CaptureEncodedVideoFrameResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.CaptureEncodedVideoFrameResponse", + () => [ + { no: 1, name: "accepted", kind: "scalar", T: 8 /* ScalarType.BOOL */, req: true }, + ], +); + +/** + * Consumes feedback accumulated by the pre-encoded passthrough encoder since + * the previous call. + * + * @generated from message livekit.proto.TakeEncodedVideoSourceFeedbackRequest + */ +const TakeEncodedVideoSourceFeedbackRequest = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.TakeEncodedVideoSourceFeedbackRequest", + () => [ + { no: 1, name: "source_handle", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + ], +); + +/** + * @generated from message livekit.proto.EncodedRateControl + */ +const EncodedRateControl = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.EncodedRateControl", + () => [ + { no: 1, name: "target_bitrate_bps", kind: "scalar", T: 4 /* ScalarType.UINT64 */, req: true }, + { no: 2, name: "framerate_fps", kind: "scalar", T: 1 /* ScalarType.DOUBLE */, req: true }, + ], +); + +/** + * @generated from message livekit.proto.TakeEncodedVideoSourceFeedbackResponse + */ +const TakeEncodedVideoSourceFeedbackResponse = /*@__PURE__*/ proto2.makeMessageType( + "livekit.proto.TakeEncodedVideoSourceFeedbackResponse", + () => [ + { no: 1, name: "keyframe_requested", kind: "scalar", T: 8 /* ScalarType.BOOL */, req: true }, + { no: 2, name: "rate_control", kind: "message", T: EncodedRateControl, opt: true }, + ], +); + /** * @generated from message livekit.proto.VideoConvertRequest */ @@ -373,6 +465,7 @@ const OwnedVideoSource = /*@__PURE__*/ proto2.makeMessageType( exports.VideoCodec = VideoCodec; +exports.EncodedFrameType = EncodedFrameType; exports.VideoRotation = VideoRotation; exports.VideoBufferType = VideoBufferType; exports.VideoStreamType = VideoStreamType; @@ -385,6 +478,12 @@ exports.NewVideoSourceRequest = NewVideoSourceRequest; exports.NewVideoSourceResponse = NewVideoSourceResponse; exports.CaptureVideoFrameRequest = CaptureVideoFrameRequest; exports.CaptureVideoFrameResponse = CaptureVideoFrameResponse; +exports.EncodedVideoBufferInfo = EncodedVideoBufferInfo; +exports.CaptureEncodedVideoFrameRequest = CaptureEncodedVideoFrameRequest; +exports.CaptureEncodedVideoFrameResponse = CaptureEncodedVideoFrameResponse; +exports.TakeEncodedVideoSourceFeedbackRequest = TakeEncodedVideoSourceFeedbackRequest; +exports.EncodedRateControl = EncodedRateControl; +exports.TakeEncodedVideoSourceFeedbackResponse = TakeEncodedVideoSourceFeedbackResponse; exports.VideoConvertRequest = VideoConvertRequest; exports.VideoConvertResponse = VideoConvertResponse; exports.VideoResolution = VideoResolution; diff --git a/livekit-ffi/protocol/ffi.proto b/livekit-ffi/protocol/ffi.proto index 53ea782bd..5bb5bf510 100644 --- a/livekit-ffi/protocol/ffi.proto +++ b/livekit-ffi/protocol/ffi.proto @@ -170,6 +170,10 @@ message FfiRequest { DefineSchemaRequest define_schema = 85; GetSchemaRequest get_schema = 86; + // Pre-encoded video + CaptureEncodedVideoFrameRequest capture_encoded_video_frame = 87; + TakeEncodedVideoSourceFeedbackRequest take_encoded_video_source_feedback = 88; + // Reconnection / chaos testing SimulateScenarioRequest simulate_scenario = 76; @@ -184,7 +188,7 @@ message FfiRequest { // Room event ready signal ReadyForRoomEventRequest ready_for_room_event = 83; - // NEXT_ID: 87 + // NEXT_ID: 89 } } @@ -299,6 +303,10 @@ message FfiResponse { DefineSchemaResponse define_schema = 85; GetSchemaResponse get_schema = 86; + // Pre-encoded video + CaptureEncodedVideoFrameResponse capture_encoded_video_frame = 87; + TakeEncodedVideoSourceFeedbackResponse take_encoded_video_source_feedback = 88; + // Reconnection / chaos testing SimulateScenarioResponse simulate_scenario = 75; @@ -313,7 +321,7 @@ message FfiResponse { // Room event ready signal ReadyForRoomEventResponse ready_for_room_event = 82; - // NEXT_ID: 87 + // NEXT_ID: 89 } } diff --git a/livekit-ffi/protocol/room.proto b/livekit-ffi/protocol/room.proto index df08ed5be..ab1068a56 100644 --- a/livekit-ffi/protocol/room.proto +++ b/livekit-ffi/protocol/room.proto @@ -335,6 +335,7 @@ enum VideoEncoderBackend { ENCODER_BACKEND_NVENC = 3; ENCODER_BACKEND_VAAPI = 4; ENCODER_BACKEND_VIDEOTOOLBOX = 5; + ENCODER_BACKEND_PRE_ENCODED = 6; } // Controls how the encoder degrades quality when bandwidth is constrained. diff --git a/livekit-ffi/protocol/video_frame.proto b/livekit-ffi/protocol/video_frame.proto index a4b610e0e..57ed26881 100644 --- a/livekit-ffi/protocol/video_frame.proto +++ b/livekit-ffi/protocol/video_frame.proto @@ -82,6 +82,43 @@ message CaptureVideoFrameRequest { message CaptureVideoFrameResponse {} +// Foreign-memory buffer containing one encoded video access unit. +// The payload pointer only needs to remain valid for the duration of the +// synchronous FFI request; the native implementation copies it before +// returning. +message EncodedVideoBufferInfo { + required uint64 data_ptr = 1; + required uint64 data_len = 2; +} + +// Push one complete pre-encoded access unit to an encoded VideoSource. +message CaptureEncodedVideoFrameRequest { + required uint64 source_handle = 1; + required EncodedVideoBufferInfo buffer = 2; + required VideoCodec codec = 3; + required EncodedFrameType frame_type = 4; + required uint32 width = 5; + required uint32 height = 6; + required int64 timestamp_us = 7; + optional FrameMetadata metadata = 8; +} + +message CaptureEncodedVideoFrameResponse { required bool accepted = 1; } + +// Consumes feedback accumulated by the pre-encoded passthrough encoder since +// the previous call. +message TakeEncodedVideoSourceFeedbackRequest { required uint64 source_handle = 1; } + +message EncodedRateControl { + required uint64 target_bitrate_bps = 1; + required double framerate_fps = 2; +} + +message TakeEncodedVideoSourceFeedbackResponse { + required bool keyframe_requested = 1; + optional EncodedRateControl rate_control = 2; +} + message VideoConvertRequest { optional bool flip_y = 1; required VideoBufferInfo buffer = 2; @@ -112,6 +149,11 @@ enum VideoCodec { H265 = 4; } +enum EncodedFrameType { + ENCODED_FRAME_KEY = 0; + ENCODED_FRAME_DELTA = 1; +} + enum VideoRotation { VIDEO_ROTATION_0 = 0; VIDEO_ROTATION_90 = 1; @@ -207,6 +249,7 @@ message VideoSourceResolution { enum VideoSourceType { VIDEO_SOURCE_NATIVE = 0; + VIDEO_SOURCE_ENCODED = 1; } message VideoSourceInfo { diff --git a/livekit-ffi/src/conversion/room.rs b/livekit-ffi/src/conversion/room.rs index 8372fc638..b2deb66bf 100644 --- a/livekit-ffi/src/conversion/room.rs +++ b/livekit-ffi/src/conversion/room.rs @@ -63,6 +63,9 @@ fn video_encoder_from_proto(backend: Option) -> Option proto::VideoEncoderBackend::EncoderBackendVideotoolbox => { Some(VideoEncoderBackend::VideoToolbox) } + proto::VideoEncoderBackend::EncoderBackendPreEncoded => { + Some(VideoEncoderBackend::PreEncoded) + } } } @@ -448,6 +451,7 @@ mod tests { proto::VideoEncoderBackend::EncoderBackendVideotoolbox, VideoEncoderBackend::VideoToolbox, ), + (proto::VideoEncoderBackend::EncoderBackendPreEncoded, VideoEncoderBackend::PreEncoded), ]; for (proto_backend, expected) in cases { diff --git a/livekit-ffi/src/server/requests.rs b/livekit-ffi/src/server/requests.rs index 8ee569885..103a2bde8 100644 --- a/livekit-ffi/src/server/requests.rs +++ b/livekit-ffi/src/server/requests.rs @@ -520,6 +520,52 @@ unsafe fn on_capture_video_frame( Ok(proto::CaptureVideoFrameResponse::default()) } +/// Push one complete pre-encoded access unit to an encoded video source. +unsafe fn on_capture_encoded_video_frame( + server: &'static FfiServer, + mut capture: proto::CaptureEncodedVideoFrameRequest, +) -> FfiResult { + let source = server.retrieve_handle::(capture.source_handle)?; + let metadata = capture.metadata.take(); + // SAFETY: `livekit_ffi_request` requires all pointers embedded in the + // decoded request to remain valid for the duration of the call. + let payload = unsafe { capture.buffer.as_slice()? }; + let accepted = source.capture_encoded_frame(&capture, payload, metadata)?; + Ok(proto::CaptureEncodedVideoFrameResponse { accepted }) +} + +impl proto::EncodedVideoBufferInfo { + /// Views the foreign encoded-video buffer as a byte slice. + /// + /// # Safety + /// + /// `data_ptr` must address `data_len` readable bytes for the returned + /// slice's lifetime. + unsafe fn as_slice(&self) -> FfiResult<&[u8]> { + let len = usize::try_from(self.data_len).map_err(|_| { + FfiError::InvalidRequest("encoded frame payload length does not fit usize".into()) + })?; + if self.data_ptr == 0 { + return Err(FfiError::InvalidRequest( + "encoded frame payload pointer must be non-null".into(), + )); + } + + // SAFETY: The caller guarantees that the pointer addresses `len` + // readable bytes for the lifetime of the returned slice. + Ok(unsafe { std::slice::from_raw_parts(self.data_ptr as *const u8, len) }) + } +} + +/// Consume pending keyframe and rate-control feedback for an encoded source. +fn on_take_encoded_video_source_feedback( + server: &'static FfiServer, + request: proto::TakeEncodedVideoSourceFeedbackRequest, +) -> FfiResult { + let source = server.retrieve_handle::(request.source_handle)?; + source.take_encoded_feedback() +} + /// Convert a video frame /// /// # Safety: The user must ensure that the pointers/len provided are valid @@ -1390,6 +1436,12 @@ pub fn handle_request( } Request::NewVideoSource(req) => on_new_video_source(server, req)?.into(), Request::CaptureVideoFrame(req) => unsafe { on_capture_video_frame(server, req)?.into() }, + Request::CaptureEncodedVideoFrame(req) => unsafe { + on_capture_encoded_video_frame(server, req)?.into() + }, + Request::TakeEncodedVideoSourceFeedback(req) => { + on_take_encoded_video_source_feedback(server, req)?.into() + } Request::VideoConvert(req) => unsafe { on_video_convert(server, req)?.into() }, Request::NewAudioStream(req) => on_new_audio_stream(server, req)?.into(), Request::NewAudioSource(req) => on_new_audio_source(server, req)?.into(), diff --git a/livekit-ffi/src/server/video_source.rs b/livekit-ffi/src/server/video_source.rs index ee1410a72..ae4a502e1 100644 --- a/livekit-ffi/src/server/video_source.rs +++ b/livekit-ffi/src/server/video_source.rs @@ -16,7 +16,9 @@ use super::{colorcvt, FfiHandle}; use crate::{proto, server, FfiError, FfiHandleId, FfiResult}; use livekit::webrtc::{ prelude::*, - video_frame::{FrameMetadata, VideoFrame}, + video_frame::{ + EncodedFrameType, EncodedVideoCodec, EncodedVideoFrame, FrameMetadata, VideoFrame, + }, }; pub struct FfiVideoSource { @@ -41,6 +43,23 @@ fn frame_metadata_from_proto(metadata: Option) -> Option EncodedVideoCodec { + match codec { + proto::VideoCodec::H264 => EncodedVideoCodec::H264, + proto::VideoCodec::H265 => EncodedVideoCodec::H265, + proto::VideoCodec::Vp8 => EncodedVideoCodec::VP8, + proto::VideoCodec::Vp9 => EncodedVideoCodec::VP9, + proto::VideoCodec::Av1 => EncodedVideoCodec::AV1, + } +} + +fn encoded_frame_type_from_proto(frame_type: proto::EncodedFrameType) -> EncodedFrameType { + match frame_type { + proto::EncodedFrameType::EncodedFrameKey => EncodedFrameType::Key, + proto::EncodedFrameType::EncodedFrameDelta => EncodedFrameType::Delta, + } +} + impl FfiVideoSource { pub fn setup( server: &'static server::FfiServer, @@ -58,6 +77,13 @@ impl FfiVideoSource { NativeVideoSource::new(new_source.resolution.into(), is_screencast); RtcVideoSource::Native(video_source) } + #[cfg(not(target_arch = "wasm32"))] + proto::VideoSourceType::VideoSourceEncoded => { + use livekit::webrtc::video_source::native::NativeVideoSource; + + let video_source = NativeVideoSource::new_encoded(new_source.resolution.into()); + RtcVideoSource::Native(video_source) + } _ => return Err(FfiError::InvalidRequest("unsupported video source type".into())), }; @@ -77,6 +103,11 @@ impl FfiVideoSource { _server: &'static server::FfiServer, capture: proto::CaptureVideoFrameRequest, ) -> FfiResult<()> { + if self.source_type != proto::VideoSourceType::VideoSourceNative { + return Err(FfiError::InvalidRequest( + "raw frames require a native video source".into(), + )); + } match self.source { #[cfg(not(target_arch = "wasm32"))] RtcVideoSource::Native(ref source) => { @@ -94,12 +125,83 @@ impl FfiVideoSource { } Ok(()) } + + pub fn capture_encoded_frame( + &self, + capture: &proto::CaptureEncodedVideoFrameRequest, + payload: &[u8], + metadata: Option, + ) -> FfiResult { + if self.source_type != proto::VideoSourceType::VideoSourceEncoded { + return Err(FfiError::InvalidRequest( + "encoded frames require an encoded video source".into(), + )); + } + if capture.width == 0 || capture.height == 0 { + return Err(FfiError::InvalidRequest( + "encoded frame dimensions must be non-zero".into(), + )); + } + let codec = proto::VideoCodec::try_from(capture.codec) + .map(encoded_video_codec_from_proto) + .map_err(|_| FfiError::InvalidRequest("unknown encoded video codec".into()))?; + let frame_type = proto::EncodedFrameType::try_from(capture.frame_type) + .map(encoded_frame_type_from_proto) + .map_err(|_| FfiError::InvalidRequest("unknown encoded frame type".into()))?; + let frame = EncodedVideoFrame { + codec, + payload, + timestamp_us: capture.timestamp_us, + frame_type, + resolution: VideoResolution { width: capture.width, height: capture.height }, + frame_metadata: frame_metadata_from_proto(metadata), + }; + + match self.source { + #[cfg(not(target_arch = "wasm32"))] + RtcVideoSource::Native(ref source) => Ok(source.capture_encoded_frame(&frame)), + _ => Err(FfiError::InvalidRequest( + "encoded video sources are unavailable on this platform".into(), + )), + } + } + + pub fn take_encoded_feedback( + &self, + ) -> FfiResult { + if self.source_type != proto::VideoSourceType::VideoSourceEncoded { + return Err(FfiError::InvalidRequest( + "encoded feedback requires an encoded video source".into(), + )); + } + + match self.source { + #[cfg(not(target_arch = "wasm32"))] + RtcVideoSource::Native(ref source) => { + let rate_control = + source.take_rate_control_request().map(|request| proto::EncodedRateControl { + target_bitrate_bps: request.target_bitrate_bps, + framerate_fps: request.framerate_fps, + }); + Ok(proto::TakeEncodedVideoSourceFeedbackResponse { + keyframe_requested: source.take_keyframe_request(), + rate_control, + }) + } + _ => Err(FfiError::InvalidRequest( + "encoded video sources are unavailable on this platform".into(), + )), + } + } } #[cfg(test)] mod tests { - use super::frame_metadata_from_proto; + use super::{ + encoded_frame_type_from_proto, encoded_video_codec_from_proto, frame_metadata_from_proto, + }; use crate::proto; + use livekit::webrtc::video_frame::{EncodedFrameType, EncodedVideoCodec}; #[test] fn empty_proto_frame_metadata_is_ignored() { @@ -119,4 +221,30 @@ mod tests { assert_eq!(metadata.frame_id, Some(456)); assert_eq!(metadata.user_data, Some(vec![7, 8, 9])); } + + #[test] + fn encoded_video_codecs_map_to_rust_types() { + let cases = [ + (proto::VideoCodec::H264, EncodedVideoCodec::H264), + (proto::VideoCodec::H265, EncodedVideoCodec::H265), + (proto::VideoCodec::Vp8, EncodedVideoCodec::VP8), + (proto::VideoCodec::Vp9, EncodedVideoCodec::VP9), + (proto::VideoCodec::Av1, EncodedVideoCodec::AV1), + ]; + for (proto_codec, expected) in cases { + assert_eq!(encoded_video_codec_from_proto(proto_codec), expected); + } + } + + #[test] + fn encoded_frame_types_map_to_rust_types() { + assert_eq!( + encoded_frame_type_from_proto(proto::EncodedFrameType::EncodedFrameKey), + EncodedFrameType::Key + ); + assert_eq!( + encoded_frame_type_from_proto(proto::EncodedFrameType::EncodedFrameDelta), + EncodedFrameType::Delta + ); + } }