Add pre-encoded video ingest FFI

.changeset/pre-encoded-ingest.md

@@ -0,0 +1,5 @@
+---
+livekit-ffi: patch
+---
+
+Add FFI support for publishing pre-encoded video frames and consuming encoder feedback.

livekit-ffi-node-bindings/proto/ffi_pb.d.ts

@@ -21,7 +21,7 @@ import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialM
 import { Message, proto2 } from "@bufbuild/protobuf";
 import type { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, ReadyForRoomEventRequest, ReadyForRoomEventResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, SimulateScenarioCallback, SimulateScenarioRequest, SimulateScenarioResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } from "./room_pb.js";
 import type { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } from "./track_pb.js";
-import type { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js";
+import type { CaptureEncodedVideoFrameRequest, CaptureEncodedVideoFrameResponse, CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, TakeEncodedVideoSourceFeedbackRequest, TakeEncodedVideoSourceFeedbackResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } from "./video_frame_pb.js";
 import type { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse, StartRecordingRequest, StartRecordingResponse, StopRecordingRequest, StopRecordingResponse } from "./audio_frame_pb.js";
 import type { E2eeRequest, E2eeResponse } from "./e2ee_pb.js";
 import type { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } from "./rpc_pb.js";
@@ -557,6 +557,20 @@ export declare class FfiRequest extends Message<FfiRequest> {
      */
     value: GetSchemaRequest;
     case: "getSchema";
+  } | {
+    /**
+     * Pre-encoded video
+     *
+     * @generated from field: livekit.proto.CaptureEncodedVideoFrameRequest capture_encoded_video_frame = 87;
+     */
+    value: CaptureEncodedVideoFrameRequest;
+    case: "captureEncodedVideoFrame";
+  } | {
+    /**
+     * @generated from field: livekit.proto.TakeEncodedVideoSourceFeedbackRequest take_encoded_video_source_feedback = 88;
+     */
+    value: TakeEncodedVideoSourceFeedbackRequest;
+    case: "takeEncodedVideoSourceFeedback";
   } | {
     /**
      * Reconnection / chaos testing
@@ -1119,6 +1133,20 @@ export declare class FfiResponse extends Message<FfiResponse> {
      */
     value: GetSchemaResponse;
     case: "getSchema";
+  } | {
+    /**
+     * Pre-encoded video
+     *
+     * @generated from field: livekit.proto.CaptureEncodedVideoFrameResponse capture_encoded_video_frame = 87;
+     */
+    value: CaptureEncodedVideoFrameResponse;
+    case: "captureEncodedVideoFrame";
+  } | {
+    /**
+     * @generated from field: livekit.proto.TakeEncodedVideoSourceFeedbackResponse take_encoded_video_source_feedback = 88;
+     */
+    value: TakeEncodedVideoSourceFeedbackResponse;
+    case: "takeEncodedVideoSourceFeedback";
   } | {
     /**
      * Reconnection / chaos testing

livekit-ffi-node-bindings/proto/ffi_pb.js

@@ -23,7 +23,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const { proto2 } = require("@bufbuild/protobuf");
 const { ConnectCallback, ConnectRequest, ConnectResponse, DisconnectCallback, DisconnectRequest, DisconnectResponse, EditChatMessageRequest, GetSessionStatsCallback, GetSessionStatsRequest, GetSessionStatsResponse, PublishDataCallback, PublishDataRequest, PublishDataResponse, PublishSipDtmfCallback, PublishSipDtmfRequest, PublishSipDtmfResponse, PublishTrackCallback, PublishTrackRequest, PublishTrackResponse, PublishTranscriptionCallback, PublishTranscriptionRequest, PublishTranscriptionResponse, ReadyForRoomEventRequest, ReadyForRoomEventResponse, RoomEvent, SendChatMessageCallback, SendChatMessageRequest, SendChatMessageResponse, SendStreamChunkCallback, SendStreamChunkRequest, SendStreamChunkResponse, SendStreamHeaderCallback, SendStreamHeaderRequest, SendStreamHeaderResponse, SendStreamTrailerCallback, SendStreamTrailerRequest, SendStreamTrailerResponse, SetDataChannelBufferedAmountLowThresholdRequest, SetDataChannelBufferedAmountLowThresholdResponse, SetLocalAttributesCallback, SetLocalAttributesRequest, SetLocalAttributesResponse, SetLocalMetadataCallback, SetLocalMetadataRequest, SetLocalMetadataResponse, SetLocalNameCallback, SetLocalNameRequest, SetLocalNameResponse, SetSubscribedRequest, SetSubscribedResponse, SimulateScenarioCallback, SimulateScenarioRequest, SimulateScenarioResponse, UnpublishTrackCallback, UnpublishTrackRequest, UnpublishTrackResponse } = require("./room_pb.js");
 const { CreateAudioTrackRequest, CreateAudioTrackResponse, CreateVideoTrackRequest, CreateVideoTrackResponse, EnableRemoteTrackRequest, EnableRemoteTrackResponse, GetStatsCallback, GetStatsRequest, GetStatsResponse, LocalTrackMuteRequest, LocalTrackMuteResponse, SetTrackSubscriptionPermissionsRequest, SetTrackSubscriptionPermissionsResponse, TrackEvent } = require("./track_pb.js");
-const { CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } = require("./video_frame_pb.js");
+const { CaptureEncodedVideoFrameRequest, CaptureEncodedVideoFrameResponse, CaptureVideoFrameRequest, CaptureVideoFrameResponse, NewVideoSourceRequest, NewVideoSourceResponse, NewVideoStreamRequest, NewVideoStreamResponse, TakeEncodedVideoSourceFeedbackRequest, TakeEncodedVideoSourceFeedbackResponse, VideoConvertRequest, VideoConvertResponse, VideoStreamEvent, VideoStreamFromParticipantRequest, VideoStreamFromParticipantResponse } = require("./video_frame_pb.js");
 const { ApmProcessReverseStreamRequest, ApmProcessReverseStreamResponse, ApmProcessStreamRequest, ApmProcessStreamResponse, ApmSetStreamDelayRequest, ApmSetStreamDelayResponse, AudioStreamEvent, AudioStreamFromParticipantRequest, AudioStreamFromParticipantResponse, CaptureAudioFrameCallback, CaptureAudioFrameRequest, CaptureAudioFrameResponse, ClearAudioBufferRequest, ClearAudioBufferResponse, FlushSoxResamplerRequest, FlushSoxResamplerResponse, GetAudioDevicesRequest, GetAudioDevicesResponse, LoadAudioFilterPluginRequest, LoadAudioFilterPluginResponse, NewApmRequest, NewApmResponse, NewAudioResamplerRequest, NewAudioResamplerResponse, NewAudioSourceRequest, NewAudioSourceResponse, NewAudioStreamRequest, NewAudioStreamResponse, NewPlatformAudioRequest, NewPlatformAudioResponse, NewSoxResamplerRequest, NewSoxResamplerResponse, PushSoxResamplerRequest, PushSoxResamplerResponse, RemixAndResampleRequest, RemixAndResampleResponse, SetPlayoutDeviceRequest, SetPlayoutDeviceResponse, SetRecordingDeviceRequest, SetRecordingDeviceResponse, StartRecordingRequest, StartRecordingResponse, StopRecordingRequest, StopRecordingResponse } = require("./audio_frame_pb.js");
 const { E2eeRequest, E2eeResponse } = require("./e2ee_pb.js");
 const { PerformRpcCallback, PerformRpcRequest, PerformRpcResponse, RegisterRpcMethodRequest, RegisterRpcMethodResponse, RpcMethodInvocationEvent, RpcMethodInvocationResponseRequest, RpcMethodInvocationResponseResponse, UnregisterRpcMethodRequest, UnregisterRpcMethodResponse } = require("./rpc_pb.js");
@@ -131,6 +131,8 @@ const FfiRequest = /*@__PURE__*/ proto2.makeMessageType(
     { no: 84, name: "remote_data_track_set_pipeline_options", kind: "message", T: RemoteDataTrackSetPipelineOptionsRequest, oneof: "message" },
     { no: 85, name: "define_schema", kind: "message", T: DefineSchemaRequest, oneof: "message" },
     { no: 86, name: "get_schema", kind: "message", T: GetSchemaRequest, oneof: "message" },
+    { no: 87, name: "capture_encoded_video_frame", kind: "message", T: CaptureEncodedVideoFrameRequest, oneof: "message" },
+    { no: 88, name: "take_encoded_video_source_feedback", kind: "message", T: TakeEncodedVideoSourceFeedbackRequest, oneof: "message" },
     { no: 76, name: "simulate_scenario", kind: "message", T: SimulateScenarioRequest, oneof: "message" },
     { no: 77, name: "new_platform_audio", kind: "message", T: NewPlatformAudioRequest, oneof: "message" },
     { no: 78, name: "get_audio_devices", kind: "message", T: GetAudioDevicesRequest, oneof: "message" },
@@ -226,6 +228,8 @@ const FfiResponse = /*@__PURE__*/ proto2.makeMessageType(
     { no: 84, name: "remote_data_track_set_pipeline_options", kind: "message", T: RemoteDataTrackSetPipelineOptionsResponse, oneof: "message" },
     { no: 85, name: "define_schema", kind: "message", T: DefineSchemaResponse, oneof: "message" },
     { no: 86, name: "get_schema", kind: "message", T: GetSchemaResponse, oneof: "message" },
+    { no: 87, name: "capture_encoded_video_frame", kind: "message", T: CaptureEncodedVideoFrameResponse, oneof: "message" },
+    { no: 88, name: "take_encoded_video_source_feedback", kind: "message", T: TakeEncodedVideoSourceFeedbackResponse, oneof: "message" },
     { no: 75, name: "simulate_scenario", kind: "message", T: SimulateScenarioResponse, oneof: "message" },
     { no: 76, name: "new_platform_audio", kind: "message", T: NewPlatformAudioResponse, oneof: "message" },
     { no: 77, name: "get_audio_devices", kind: "message", T: GetAudioDevicesResponse, oneof: "message" },

livekit-ffi-node-bindings/proto/room_pb.d.ts

@@ -125,6 +125,11 @@ export declare enum VideoEncoderBackend {
    * @generated from enum value: ENCODER_BACKEND_VIDEOTOOLBOX = 5;
    */
   ENCODER_BACKEND_VIDEOTOOLBOX = 5,
+
+  /**
+   * @generated from enum value: ENCODER_BACKEND_PRE_ENCODED = 6;
+   */
+  ENCODER_BACKEND_PRE_ENCODED = 6,
 }
 
 /**

livekit-ffi-node-bindings/proto/room_pb.js

@@ -65,6 +65,7 @@ const VideoEncoderBackend = /*@__PURE__*/ proto2.makeEnum(
     {no: 3, name: "ENCODER_BACKEND_NVENC"},
     {no: 4, name: "ENCODER_BACKEND_VAAPI"},
     {no: 5, name: "ENCODER_BACKEND_VIDEOTOOLBOX"},
+    {no: 6, name: "ENCODER_BACKEND_PRE_ENCODED"},
   ],
 );