From 2f39e70b6760fd5c663530b950388d98d23266f6 Mon Sep 17 00:00:00 2001 From: Joshua Goins Date: Fri, 22 Nov 2024 08:53:03 -0500 Subject: [PATCH] Update to latest LiveKit, fix up QML --- src/CMakeLists.txt | 7 +- src/calls/callcontroller.cpp | 32 ++- src/calls/callcontroller.h | 4 +- src/protocols/audio_frame.proto | 174 +++++++++--- src/protocols/e2ee.proto | 64 ++--- src/protocols/ffi.proto | 148 +++++++---- src/protocols/handle.proto | 4 +- src/protocols/participant.proto | 24 +- src/protocols/room.proto | 443 +++++++++++++++++++++---------- src/protocols/rpc.proto | 81 ++++++ src/protocols/stats.proto | 456 ++++++++++++++++---------------- src/protocols/track.proto | 82 +++--- src/protocols/video_frame.proto | 100 ++++--- src/qml/CallPage.qml | 75 ++---- 14 files changed, 1053 insertions(+), 641 deletions(-) create mode 100644 src/protocols/rpc.proto diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index adf472493..f9e03c5ac 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -411,10 +411,6 @@ if (NOT ANDROID AND NOT WIN32 AND NOT APPLE AND NOT HAIKU) target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER) target_compile_definitions(neochat PUBLIC -DHAVE_X11=1) target_sources(neochat PRIVATE runner.cpp) -if (NOT ANDROID AND NOT WIN32 AND NOT APPLE) - #target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER) - target_compile_definitions(neochat PUBLIC -DHAVE_X11) - #target_sources(neochat PRIVATE runner.cpp) if (TARGET KUnifiedPush) target_sources(neochat PRIVATE fakerunner.cpp) @@ -435,9 +431,10 @@ qt_add_protobuf(neochat protocols/participant.proto protocols/stats.proto protocols/track.proto + protocols/rpc.proto ) -target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums) +target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums ${CMAKE_CURRENT_SOURCE_DIR}/calls) target_link_libraries(neochat PRIVATE settingsplugin timelineplugin devtoolsplugin loginplugin chatbarplugin) target_link_libraries(neochat PUBLIC Qt::Core diff --git a/src/calls/callcontroller.cpp b/src/calls/callcontroller.cpp index 36cf7b485..6b1c5b2c2 100644 --- a/src/calls/callcontroller.cpp +++ b/src/calls/callcontroller.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include @@ -29,7 +28,7 @@ using namespace livekit::proto; using namespace Quotient; extern "C" { -void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs); +void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs, const char *, const char *); } void callback(const uint8_t *data, size_t length) @@ -50,7 +49,7 @@ CallController::CallController() void CallController::init() { qRegisterProtobufTypes(); - livekit_ffi_initialize(callback, true); + livekit_ffi_initialize(callback, true, "test", "1.0"); } static void handleLog(LogRecordRepeated &&logs) @@ -65,15 +64,16 @@ static void handleLog(LogRecordRepeated &&logs) void CallController::handleConnect(ConnectCallback &&callback) { - qWarning() << "Connecting to" << callback.room().info().name() << "with id" << callback.asyncId(); + qWarning() << "Connecting to" << callback.result().room().info().name() << "with id" << callback.asyncId(); if (!m_connectingRooms.contains(callback.asyncId()) || !m_connectingRooms[callback.asyncId()] || m_connectingRooms[callback.asyncId()]->id() != callback.room().info().name()) { qWarning() << "Connecting to unexpected room"; return; } m_connectingRooms.remove(callback.asyncId()); - m_rooms[callback.asyncId()] = callback.room(); - localParticipant = callback.localParticipant().handle().id_proto(); + m_rooms[callback.asyncId()] = callback.result().room(); + localParticipant = callback.result().localParticipant().handle().id_proto(); + Q_EMIT connected(); } void CallController::handleDispose(DisposeCallback &&callback) @@ -92,7 +92,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event) if (event.hasParticipantConnected()) { qWarning() << "Participant connected" << event.participantConnected().info().info().identity(); } else if (event.hasParticipantDisconnected()) { - qWarning() << "Participant connected" << event.participantDisconnected().participantSid(); + qWarning() << "Participant connected" << event.participantDisconnected().participantIdentity(); } else if (event.hasLocalTrackPublished()) { qWarning() << "Local track published"; m_localVideoTrackSid = event.localTrackPublished().trackSid(); @@ -140,7 +140,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event) } else if (event.hasTrackUnmuted()) { qWarning() << "Track unmuted"; } else if (event.hasActiveSpeakersChanged()) { - // qWarning() << "Active speakers changed"; + qWarning() << "Active speakers changed"; } else if (event.hasRoomMetadataChanged()) { qWarning() << "room metadata changed"; } else if (event.hasParticipantMetadataChanged()) { @@ -265,10 +265,6 @@ void CallController::handleEvent(FfiEvent &&event) qWarning() << "publish data"; } else if (event.hasCaptureAudioFrame()) { qWarning() << "audio frame"; - } else if (event.hasUpdateLocalMetadata()) { - qWarning() << "update local metadata"; - } else if (event.hasUpdateLocalName()) { - qWarning() << "update local name"; } else if (event.hasGetStats()) { qWarning() << "get stats"; } else if (event.hasGetSessionStats()) { @@ -297,11 +293,11 @@ void CallController::handleCallMemberEvent(const Quotient::CallMemberEvent *even {"device_id"_ls, connection->deviceId()}, }) .toJson(); - if (event->memberships().isEmpty()) { + // This is an old event! + if (!event->contentJson().contains("foci_preferred"_ls)) { return; } - auto membership = event->memberships()[0].toObject(); - QNetworkRequest request(QUrl((membership["foci_active"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls))); + QNetworkRequest request(QUrl((event->contentJson()["foci_preferred"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls))); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"_ls); auto reply = nam->post(request, json); connect(reply, &QNetworkReply::finished, this, [reply, this, room]() { @@ -363,6 +359,7 @@ void CallController::setCameraVideoSink(QVideoSink *videoSink) ffiRequest.setNewVideoSource(newVideoSourceRequest); auto response = request(std::move(ffiRequest)); handle = response.newVideoSource().source().handle().id_proto(); + m_localVideoTrackHandle = handle; CreateVideoTrackRequest createVideoTrackRequest; createVideoTrackRequest.setName("Camera"_ls); @@ -410,7 +407,7 @@ void CallController::setVideoSink(QObject *sink) void LivekitVideoSink::setVideoSink(QVideoSink *videoSink) { m_videoSink = videoSink; - CallController::instance().setCameraVideoSink(videoSink); + CallController::instance().setVideoSink(videoSink); Q_EMIT videoSinkChanged(); } QVideoSink *LivekitVideoSink::videoSink() const @@ -437,10 +434,11 @@ void CallController::publishTrack(uint64_t id) { PublishTrackRequest publishTrackRequest; - publishTrackRequest.setTrackHandle(id); + publishTrackRequest.setTrackHandle(m_localVideoTrackHandle); publishTrackRequest.setLocalParticipantHandle(localParticipant); TrackPublishOptions options; options.setSource(TrackSourceGadget::SOURCE_CAMERA); + options.setVideoCodec(VideoCodecGadget::VideoCodec::VP8); publishTrackRequest.setOptions(options); auto request = FfiRequest(); diff --git a/src/calls/callcontroller.h b/src/calls/callcontroller.h index 8f731f740..dab0c4694 100644 --- a/src/calls/callcontroller.h +++ b/src/calls/callcontroller.h @@ -48,12 +48,13 @@ public: // Internal. Do not use. void handleEvent(livekit::proto::FfiEvent &&event); Q_INVOKABLE void setVideoSink(QObject *sink); - void setCameraVideoSink(QVideoSink *videoSink); + Q_INVOKABLE void setCameraVideoSink(QVideoSink *videoSink); Q_INVOKABLE void toggleCamera(); Q_SIGNALS: void callStarted(); + void connected(); private: CallController(); @@ -74,6 +75,7 @@ private: uint64_t localParticipant = 100000; QString m_localVideoTrackSid; uint64_t m_localVideoTrackId; + uint64_t m_localVideoTrackHandle; }; class LivekitVideoSink : public QObject diff --git a/src/protocols/audio_frame.proto b/src/protocols/audio_frame.proto index f61c74c38..4d70c310f 100644 --- a/src/protocols/audio_frame.proto +++ b/src/protocols/audio_frame.proto @@ -12,76 +12,162 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; +import "track.proto"; // Create a new AudioStream // AudioStream is used to receive audio frames from a track message NewAudioStreamRequest { - uint64 track_handle = 1; - AudioStreamType type = 2; + required uint64 track_handle = 1; + required AudioStreamType type = 2; + optional uint32 sample_rate = 3; + optional uint32 num_channels = 4; } -message NewAudioStreamResponse { OwnedAudioStream stream = 1; } +message NewAudioStreamResponse { required OwnedAudioStream stream = 1; } + +message AudioStreamFromParticipantRequest { + required uint64 participant_handle = 1; + required AudioStreamType type = 2; + optional TrackSource track_source = 3; + optional uint32 sample_rate = 5; + optional uint32 num_channels = 6; +} + +message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; } // Create a new AudioSource message NewAudioSourceRequest { - AudioSourceType type = 1; + required AudioSourceType type = 1; optional AudioSourceOptions options = 2; - uint32 sample_rate = 3; - uint32 num_channels = 4; + required uint32 sample_rate = 3; + required uint32 num_channels = 4; + optional uint32 queue_size_ms = 5; } -message NewAudioSourceResponse { OwnedAudioSource source = 1; } +message NewAudioSourceResponse { required OwnedAudioSource source = 1; } // Push a frame to an AudioSource // The data provided must be available as long as the client receive the callback. message CaptureAudioFrameRequest { - uint64 source_handle = 1; - AudioFrameBufferInfo buffer = 2; + required uint64 source_handle = 1; + required AudioFrameBufferInfo buffer = 2; } message CaptureAudioFrameResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message CaptureAudioFrameCallback { - uint64 async_id = 1; + required uint64 async_id = 1; optional string error = 2; } +message ClearAudioBufferRequest { + required uint64 source_handle = 1; +} +message ClearAudioBufferResponse {} + // Create a new AudioResampler message NewAudioResamplerRequest {} message NewAudioResamplerResponse { - OwnedAudioResampler resampler = 1; + required OwnedAudioResampler resampler = 1; } // Remix and resample an audio frame message RemixAndResampleRequest { - uint64 resampler_handle = 1; - AudioFrameBufferInfo buffer = 2; - uint32 num_channels = 3; - uint32 sample_rate = 4; + required uint64 resampler_handle = 1; + required AudioFrameBufferInfo buffer = 2; + required uint32 num_channels = 3; + required uint32 sample_rate = 4; } message RemixAndResampleResponse { - OwnedAudioFrameBuffer buffer = 1; + required OwnedAudioFrameBuffer buffer = 1; } + +// New resampler using SoX (much better quality) + +message NewSoxResamplerRequest { + required double input_rate = 1; + required double output_rate = 2; + required uint32 num_channels = 3; + required SoxResamplerDataType input_data_type = 4; + required SoxResamplerDataType output_data_type = 5; + required SoxQualityRecipe quality_recipe = 6; + optional uint32 flags = 7; +} +message NewSoxResamplerResponse { + oneof message { + OwnedSoxResampler resampler = 1; + string error = 2; + } + +} + +message PushSoxResamplerRequest { + required uint64 resampler_handle = 1; + required uint64 data_ptr = 2; // *const i16 + required uint32 size = 3; // in bytes +} + +message PushSoxResamplerResponse { + required uint64 output_ptr = 1; // *const i16 (could be null) + required uint32 size = 2; // in bytes + optional string error = 3; +} + +message FlushSoxResamplerRequest { + required uint64 resampler_handle = 1; +} + +message FlushSoxResamplerResponse { + required uint64 output_ptr = 1; // *const i16 (could be null) + required uint32 size = 2; // in bytes + optional string error = 3; +} + +enum SoxResamplerDataType { + // TODO(theomonnom): support other datatypes (shouldn't really be needed) + SOXR_DATATYPE_INT16I = 0; + SOXR_DATATYPE_INT16S = 1; +} + +enum SoxQualityRecipe { + SOXR_QUALITY_QUICK = 0; + SOXR_QUALITY_LOW = 1; + SOXR_QUALITY_MEDIUM = 2; + SOXR_QUALITY_HIGH = 3; + SOXR_QUALITY_VERYHIGH = 4; +} + +enum SoxFlagBits { + SOXR_ROLLOFF_SMALL = 0; // 1 << 0 + SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1 + SOXR_ROLLOFF_NONE = 2; // 1 << 2 + SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3 + SOXR_DOUBLE_PRECISION = 4; // 1 << 4 + SOXR_VR = 5; // 1 << 5 +} + + + // // AudioFrame buffer // message AudioFrameBufferInfo { - uint64 data_ptr = 1; // *const i16 - uint32 num_channels = 2; - uint32 sample_rate = 3; - uint32 samples_per_channel = 4; + required uint64 data_ptr = 1; // *const i16 + required uint32 num_channels = 2; + required uint32 sample_rate = 3; + required uint32 samples_per_channel = 4; } message OwnedAudioFrameBuffer { - FfiOwnedHandle handle = 1; - AudioFrameBufferInfo info = 2; + required FfiOwnedHandle handle = 1; + required AudioFrameBufferInfo info = 2; } // @@ -94,16 +180,16 @@ enum AudioStreamType { } message AudioStreamInfo { - AudioStreamType type = 1; + required AudioStreamType type = 1; } message OwnedAudioStream { - FfiOwnedHandle handle = 1; - AudioStreamInfo info = 2; + required FfiOwnedHandle handle = 1; + required AudioStreamInfo info = 2; } message AudioStreamEvent { - uint64 stream_handle = 1; + required uint64 stream_handle = 1; oneof message { AudioFrameReceived frame_received = 2; AudioStreamEOS eos = 3; @@ -111,7 +197,7 @@ message AudioStreamEvent { } message AudioFrameReceived { - OwnedAudioFrameBuffer frame = 1; + required OwnedAudioFrameBuffer frame = 1; } message AudioStreamEOS {} @@ -121,9 +207,9 @@ message AudioStreamEOS {} // message AudioSourceOptions { - bool echo_cancellation = 1; - bool noise_suppression = 2; - bool auto_gain_control = 3; + required bool echo_cancellation = 1; + required bool noise_suppression = 2; + required bool auto_gain_control = 3; } enum AudioSourceType { @@ -131,12 +217,12 @@ enum AudioSourceType { } message AudioSourceInfo { - AudioSourceType type = 2; + required AudioSourceType type = 2; } message OwnedAudioSource { - FfiOwnedHandle handle = 1; - AudioSourceInfo info = 2; + required FfiOwnedHandle handle = 1; + required AudioSourceInfo info = 2; } // @@ -146,6 +232,20 @@ message OwnedAudioSource { message AudioResamplerInfo { } message OwnedAudioResampler { - FfiOwnedHandle handle = 1; - AudioResamplerInfo info = 2; + required FfiOwnedHandle handle = 1; + required AudioResamplerInfo info = 2; +} + + + +// +// Sox AudioResampler +// + + +message SoxResamplerInfo {} + +message OwnedSoxResampler { + required FfiOwnedHandle handle = 1; + required SoxResamplerInfo info = 2; } diff --git a/src/protocols/e2ee.proto b/src/protocols/e2ee.proto index 39f142bb4..bdc20140f 100644 --- a/src/protocols/e2ee.proto +++ b/src/protocols/e2ee.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -26,23 +26,23 @@ enum EncryptionType { } message FrameCryptor { - string participant_identity = 1; - string track_sid = 2; - int32 key_index = 3; - bool enabled = 4; + required string participant_identity = 1; + required string track_sid = 2; + required int32 key_index = 3; + required bool enabled = 4; } message KeyProviderOptions { // Only specify if you want to use a shared_key optional bytes shared_key = 1; - int32 ratchet_window_size = 2; - bytes ratchet_salt = 3; - int32 failure_tolerance = 4; // -1 = no tolerence + required int32 ratchet_window_size = 2; + required bytes ratchet_salt = 3; + required int32 failure_tolerance = 4; // -1 = no tolerance } message E2eeOptions { - EncryptionType encryption_type = 1; - KeyProviderOptions key_provider_options = 2; + required EncryptionType encryption_type = 1; + required KeyProviderOptions key_provider_options = 2; } enum EncryptionState { @@ -56,7 +56,7 @@ enum EncryptionState { } message E2eeManagerSetEnabledRequest { - bool enabled = 1; + required bool enabled = 1; } message E2eeManagerSetEnabledResponse {} @@ -66,64 +66,64 @@ message E2eeManagerGetFrameCryptorsResponse { } message FrameCryptorSetEnabledRequest { - string participant_identity = 1; - string track_sid = 2; - bool enabled = 3; + required string participant_identity = 1; + required string track_sid = 2; + required bool enabled = 3; } -message FrameCryptorSetEnabledResponse { } +message FrameCryptorSetEnabledResponse {} message FrameCryptorSetKeyIndexRequest { - string participant_identity = 1; - string track_sid = 2; - int32 key_index = 3; + required string participant_identity = 1; + required string track_sid = 2; + required int32 key_index = 3; } -message FrameCryptorSetKeyIndexResponse { } +message FrameCryptorSetKeyIndexResponse {} message SetSharedKeyRequest { - bytes shared_key = 1; - int32 key_index = 2; + required bytes shared_key = 1; + required int32 key_index = 2; } -message SetSharedKeyResponse { } +message SetSharedKeyResponse {} message RatchetSharedKeyRequest { - int32 key_index = 1; + required int32 key_index = 1; } message RatchetSharedKeyResponse { optional bytes new_key = 1; } message GetSharedKeyRequest { - int32 key_index = 1; + required int32 key_index = 1; } message GetSharedKeyResponse { optional bytes key = 1; } message SetKeyRequest { - string participant_identity = 1; - bytes key = 2; - int32 key_index = 3; + required string participant_identity = 1; + required bytes key = 2; + required int32 key_index = 3; } message SetKeyResponse {} message RatchetKeyRequest { - string participant_identity = 1; - int32 key_index = 2; + required string participant_identity = 1; + required int32 key_index = 2; } message RatchetKeyResponse { optional bytes new_key = 1; } message GetKeyRequest { - string participant_identity = 1; - int32 key_index = 2; + required string participant_identity = 1; + required int32 key_index = 2; } message GetKeyResponse { optional bytes key = 1; } message E2eeRequest { - uint64 room_handle = 1; + required uint64 room_handle = 1; oneof message { E2eeManagerSetEnabledRequest manager_set_enabled = 2; E2eeManagerGetFrameCryptorsRequest manager_get_frame_cryptors = 3; diff --git a/src/protocols/ffi.proto b/src/protocols/ffi.proto index b3d4f6f69..e8c1fda4a 100644 --- a/src/protocols/ffi.proto +++ b/src/protocols/ffi.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -23,6 +23,7 @@ import "track.proto"; import "room.proto"; import "video_frame.proto"; import "audio_frame.proto"; +import "rpc.proto"; // **How is the livekit-ffi working: // We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we @@ -63,28 +64,48 @@ message FfiRequest { UnpublishTrackRequest unpublish_track = 6; PublishDataRequest publish_data = 7; SetSubscribedRequest set_subscribed = 8; - UpdateLocalMetadataRequest update_local_metadata = 9; - UpdateLocalNameRequest update_local_name = 10; - GetSessionStatsRequest get_session_stats = 11; + SetLocalMetadataRequest set_local_metadata = 9; + SetLocalNameRequest set_local_name = 10; + SetLocalAttributesRequest set_local_attributes = 11; + GetSessionStatsRequest get_session_stats = 12; + PublishTranscriptionRequest publish_transcription = 13; + PublishSipDtmfRequest publish_sip_dtmf = 14; + // Track - CreateVideoTrackRequest create_video_track = 12; - CreateAudioTrackRequest create_audio_track = 13; - GetStatsRequest get_stats = 14; + CreateVideoTrackRequest create_video_track = 15; + CreateAudioTrackRequest create_audio_track = 16; + LocalTrackMuteRequest local_track_mute = 17; + EnableRemoteTrackRequest enable_remote_track = 18; + GetStatsRequest get_stats = 19; - // Video - NewVideoStreamRequest new_video_stream = 16; - NewVideoSourceRequest new_video_source = 17; - CaptureVideoFrameRequest capture_video_frame = 18; - VideoConvertRequest video_convert = 19; +// Video + NewVideoStreamRequest new_video_stream = 20; + NewVideoSourceRequest new_video_source = 21; + CaptureVideoFrameRequest capture_video_frame = 22; + VideoConvertRequest video_convert = 23; + VideoStreamFromParticipantRequest video_stream_from_participant = 24; // Audio - NewAudioStreamRequest new_audio_stream = 22; - NewAudioSourceRequest new_audio_source = 23; - CaptureAudioFrameRequest capture_audio_frame = 24; - NewAudioResamplerRequest new_audio_resampler = 25; - RemixAndResampleRequest remix_and_resample = 26; - E2eeRequest e2ee = 27; + NewAudioStreamRequest new_audio_stream = 25; + NewAudioSourceRequest new_audio_source = 26; + CaptureAudioFrameRequest capture_audio_frame = 27; + ClearAudioBufferRequest clear_audio_buffer = 28; + NewAudioResamplerRequest new_audio_resampler = 29; + RemixAndResampleRequest remix_and_resample = 30; + E2eeRequest e2ee = 31; + AudioStreamFromParticipantRequest audio_stream_from_participant = 32; + NewSoxResamplerRequest new_sox_resampler = 33; + PushSoxResamplerRequest push_sox_resampler = 34; + FlushSoxResamplerRequest flush_sox_resampler = 35; + SendChatMessageRequest send_chat_message = 36; + EditChatMessageRequest edit_chat_message = 37; + + // RPC + PerformRpcRequest perform_rpc = 38; + RegisterRpcMethodRequest register_rpc_method = 39; + UnregisterRpcMethodRequest unregister_rpc_method = 40; + RpcMethodInvocationResponseRequest rpc_method_invocation_response = 41; } } @@ -100,28 +121,45 @@ message FfiResponse { UnpublishTrackResponse unpublish_track = 6; PublishDataResponse publish_data = 7; SetSubscribedResponse set_subscribed = 8; - UpdateLocalMetadataResponse update_local_metadata = 9; - UpdateLocalNameResponse update_local_name = 10; - GetSessionStatsResponse get_session_stats = 11; + SetLocalMetadataResponse set_local_metadata = 9; + SetLocalNameResponse set_local_name = 10; + SetLocalAttributesResponse set_local_attributes = 11; + GetSessionStatsResponse get_session_stats = 12; + PublishTranscriptionResponse publish_transcription = 13; + PublishSipDtmfResponse publish_sip_dtmf = 14; // Track - CreateVideoTrackResponse create_video_track = 12; - CreateAudioTrackResponse create_audio_track = 13; - GetStatsResponse get_stats = 14; + CreateVideoTrackResponse create_video_track = 15; + CreateAudioTrackResponse create_audio_track = 16; + LocalTrackMuteResponse local_track_mute = 17; + EnableRemoteTrackResponse enable_remote_track = 18; + GetStatsResponse get_stats = 19; // Video - NewVideoStreamResponse new_video_stream = 16; - NewVideoSourceResponse new_video_source = 17; - CaptureVideoFrameResponse capture_video_frame = 18; - VideoConvertResponse video_convert = 19; + NewVideoStreamResponse new_video_stream = 20; + NewVideoSourceResponse new_video_source = 21; + CaptureVideoFrameResponse capture_video_frame = 22; + VideoConvertResponse video_convert = 23; + VideoStreamFromParticipantResponse video_stream_from_participant = 24; // Audio - NewAudioStreamResponse new_audio_stream = 22; - NewAudioSourceResponse new_audio_source = 23; - CaptureAudioFrameResponse capture_audio_frame = 24; - NewAudioResamplerResponse new_audio_resampler = 25; - RemixAndResampleResponse remix_and_resample = 26; - E2eeResponse e2ee = 27; + NewAudioStreamResponse new_audio_stream = 25; + NewAudioSourceResponse new_audio_source = 26; + CaptureAudioFrameResponse capture_audio_frame = 27; + ClearAudioBufferResponse clear_audio_buffer = 28; + NewAudioResamplerResponse new_audio_resampler = 29; + RemixAndResampleResponse remix_and_resample = 30; + AudioStreamFromParticipantResponse audio_stream_from_participant = 31; + E2eeResponse e2ee = 32; + NewSoxResamplerResponse new_sox_resampler = 33; + PushSoxResamplerResponse push_sox_resampler = 34; + FlushSoxResamplerResponse flush_sox_resampler = 35; + SendChatMessageResponse send_chat_message = 36; + // RPC + PerformRpcResponse perform_rpc = 37; + RegisterRpcMethodResponse register_rpc_method = 38; + UnregisterRpcMethodResponse unregister_rpc_method = 39; + RpcMethodInvocationResponseResponse rpc_method_invocation_response = 40; } } @@ -135,18 +173,24 @@ message FfiEvent { VideoStreamEvent video_stream_event = 3; AudioStreamEvent audio_stream_event = 4; ConnectCallback connect = 5; - DisconnectCallback disconnect = 6; - DisposeCallback dispose = 7; - PublishTrackCallback publish_track = 8; - UnpublishTrackCallback unpublish_track = 9; - PublishDataCallback publish_data = 10; - CaptureAudioFrameCallback capture_audio_frame = 11; - UpdateLocalMetadataCallback update_local_metadata = 12; - UpdateLocalNameCallback update_local_name = 13; - GetStatsCallback get_stats = 14; - LogBatch logs = 15; - GetSessionStatsCallback get_session_stats = 16; - Panic panic = 17; + DisconnectCallback disconnect = 7; + DisposeCallback dispose = 8; + PublishTrackCallback publish_track = 9; + UnpublishTrackCallback unpublish_track = 10; + PublishDataCallback publish_data = 11; + PublishTranscriptionCallback publish_transcription = 12; + CaptureAudioFrameCallback capture_audio_frame = 13; + SetLocalMetadataCallback set_local_metadata = 14; + SetLocalNameCallback set_local_name = 15; + SetLocalAttributesCallback set_local_attributes = 16; + GetStatsCallback get_stats = 17; + LogBatch logs = 18; + GetSessionStatsCallback get_session_stats = 19; + Panic panic = 20; + PublishSipDtmfCallback publish_sip_dtmf = 21; + SendChatMessageCallback chat_message = 22; + PerformRpcCallback perform_rpc = 23; + RpcMethodInvocationEvent rpc_method_invocation = 24; } } @@ -154,14 +198,14 @@ message FfiEvent { // e.g: This is used for the Unity Editor after each assemblies reload. // TODO(theomonnom): Implement a debug mode where we can find all leaked handles? message DisposeRequest { - bool async = 1; + required bool async = 1; } message DisposeResponse { optional uint64 async_id = 1; // None if sync } message DisposeCallback { - uint64 async_id = 1; + required uint64 async_id = 1; } enum LogLevel { @@ -173,12 +217,12 @@ enum LogLevel { } message LogRecord { - LogLevel level = 1; - string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc... + required LogLevel level = 1; + required string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc... optional string module_path = 3; optional string file = 4; optional uint32 line = 5; - string message = 6; + required string message = 6; } message LogBatch { @@ -186,7 +230,7 @@ message LogBatch { } message Panic { - string message = 1; + required string message = 1; } // TODO(theomonnom): Debug messages (Print handles). diff --git a/src/protocols/handle.proto b/src/protocols/handle.proto index f86ee9d30..90c74671e 100644 --- a/src/protocols/handle.proto +++ b/src/protocols/handle.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -27,5 +27,5 @@ option csharp_namespace = "LiveKit.Proto"; // When refering to a handle without owning it, we just use a uint32 without this message. // (the variable name is suffixed with "_handle") message FfiOwnedHandle { - uint64 id = 1; + required uint64 id = 1; } diff --git a/src/protocols/participant.proto b/src/protocols/participant.proto index 271eccfa4..c0a480f2a 100644 --- a/src/protocols/participant.proto +++ b/src/protocols/participant.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -20,13 +20,23 @@ option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; message ParticipantInfo { - string sid = 1; - string name = 2; - string identity = 3; - string metadata = 4; + required string sid = 1; + required string name = 2; + required string identity = 3; + required string metadata = 4; + map attributes = 5; + required ParticipantKind kind = 6; } message OwnedParticipant { - FfiOwnedHandle handle = 1; - ParticipantInfo info = 2; + required FfiOwnedHandle handle = 1; + required ParticipantInfo info = 2; +} + +enum ParticipantKind { + PARTICIPANT_KIND_STANDARD = 0; + PARTICIPANT_KIND_INGRESS = 1; + PARTICIPANT_KIND_EGRESS = 2; + PARTICIPANT_KIND_SIP = 3; + PARTICIPANT_KIND_AGENT = 4; } \ No newline at end of file diff --git a/src/protocols/room.proto b/src/protocols/room.proto index 659f378d8..edae5c007 100644 --- a/src/protocols/room.proto +++ b/src/protocols/room.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -26,147 +26,240 @@ import "stats.proto"; // Connect to a new LiveKit room message ConnectRequest { - string url = 1; - string token = 2; - RoomOptions options = 3; + required string url = 1; + required string token = 2; + required RoomOptions options = 3; } message ConnectResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message ConnectCallback { message ParticipantWithTracks { - OwnedParticipant participant = 1; + required OwnedParticipant participant = 1; // TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send // a TrackSubscribed event repeated OwnedTrackPublication publications = 2; } - uint64 async_id = 1; - optional string error = 2; - OwnedRoom room = 3; - OwnedParticipant local_participant = 4; - repeated ParticipantWithTracks participants = 5; + message Result { + required OwnedRoom room = 1; + required OwnedParticipant local_participant = 2; + repeated ParticipantWithTracks participants = 3; + } + + required uint64 async_id = 1; + oneof message { + string error = 2; + Result result = 3; + } + } // Disconnect from the a room -message DisconnectRequest { uint64 room_handle = 1; } -message DisconnectResponse { uint64 async_id = 1; } -message DisconnectCallback { uint64 async_id = 1; } +message DisconnectRequest { required uint64 room_handle = 1; } +message DisconnectResponse { required uint64 async_id = 1; } +message DisconnectCallback { required uint64 async_id = 1; } // Publish a track to the room message PublishTrackRequest { - uint64 local_participant_handle = 1; - uint64 track_handle = 2; - TrackPublishOptions options = 3; + required uint64 local_participant_handle = 1; + required uint64 track_handle = 2; + required TrackPublishOptions options = 3; } message PublishTrackResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message PublishTrackCallback { - uint64 async_id = 1; - optional string error = 2; - OwnedTrackPublication publication = 3; + required uint64 async_id = 1; + oneof message { + string error = 2; + OwnedTrackPublication publication = 3; + } + } // Unpublish a track from the room message UnpublishTrackRequest { - uint64 local_participant_handle = 1; - string track_sid = 2; - bool stop_on_unpublish = 3; + required uint64 local_participant_handle = 1; + required string track_sid = 2; + required bool stop_on_unpublish = 3; } message UnpublishTrackResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message UnpublishTrackCallback { - uint64 async_id = 1; + required uint64 async_id = 1; optional string error = 2; } // Publish data to other participants message PublishDataRequest { - uint64 local_participant_handle = 1; - uint64 data_ptr = 2; - uint64 data_len = 3; - DataPacketKind kind = 4; - repeated string destination_sids = 5; // destination + required uint64 local_participant_handle = 1; + required uint64 data_ptr = 2; + required uint64 data_len = 3; + required bool reliable = 4; + repeated string destination_sids = 5 [deprecated=true]; optional string topic = 6; + repeated string destination_identities = 7; } message PublishDataResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message PublishDataCallback { - uint64 async_id = 1; + required uint64 async_id = 1; + optional string error = 2; +} + +// Publish transcription messages to room +message PublishTranscriptionRequest { + required uint64 local_participant_handle = 1; + required string participant_identity = 2; + required string track_id = 3; + repeated TranscriptionSegment segments = 4; +} +message PublishTranscriptionResponse { + required uint64 async_id = 1; +} +message PublishTranscriptionCallback { + required uint64 async_id = 1; + optional string error = 2; +} + +// Publish Sip DTMF messages to other participants +message PublishSipDtmfRequest { + required uint64 local_participant_handle = 1; + required uint32 code = 2; + required string digit = 3; + repeated string destination_identities = 4; +} +message PublishSipDtmfResponse { + required uint64 async_id = 1; +} +message PublishSipDtmfCallback { + required uint64 async_id = 1; optional string error = 2; } // Change the local participant's metadata -message UpdateLocalMetadataRequest { - uint64 local_participant_handle = 1; - string metadata = 2; +message SetLocalMetadataRequest { + required uint64 local_participant_handle = 1; + required string metadata = 2; } -message UpdateLocalMetadataResponse { - uint64 async_id = 1; +message SetLocalMetadataResponse { + required uint64 async_id = 1; } -message UpdateLocalMetadataCallback { - uint64 async_id = 1; +message SetLocalMetadataCallback { + required uint64 async_id = 1; + optional string error = 2; +} + +message SendChatMessageRequest { + required uint64 local_participant_handle = 1; + required string message = 2; + repeated string destination_identities = 3; + optional string sender_identity = 4; +} +message EditChatMessageRequest { + required uint64 local_participant_handle = 1; + required string edit_text = 2; + required ChatMessage original_message = 3; + repeated string destination_identities = 4; + optional string sender_identity = 5; +} +message SendChatMessageResponse { + required uint64 async_id = 1; +} +message SendChatMessageCallback { + required uint64 async_id = 1; + oneof message { + string error = 2; + ChatMessage chat_message = 3; + } +} + +// Change the local participant's attributes +message SetLocalAttributesRequest { + required uint64 local_participant_handle = 1; + repeated AttributesEntry attributes = 2; +} + +message AttributesEntry { + required string key = 1; + required string value = 2; +} + +message SetLocalAttributesResponse { + required uint64 async_id = 1; +} +message SetLocalAttributesCallback { + required uint64 async_id = 1; + optional string error = 2; } // Change the local participant's name -message UpdateLocalNameRequest { - uint64 local_participant_handle = 1; - string name = 2; +message SetLocalNameRequest { + required uint64 local_participant_handle = 1; + required string name = 2; } -message UpdateLocalNameResponse { - uint64 async_id = 1; +message SetLocalNameResponse { + required uint64 async_id = 1; } -message UpdateLocalNameCallback { - uint64 async_id = 1; +message SetLocalNameCallback { + required uint64 async_id = 1; + optional string error = 2; } // Change the "desire" to subs2ribe to a track message SetSubscribedRequest { - bool subscribe = 1; - uint64 publication_handle = 2; + required bool subscribe = 1; + required uint64 publication_handle = 2; } message SetSubscribedResponse {} message GetSessionStatsRequest { - uint64 room_handle = 1; + required uint64 room_handle = 1; } message GetSessionStatsResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message GetSessionStatsCallback { - uint64 async_id = 1; - optional string error = 2; - repeated RtcStats publisher_stats = 3; - repeated RtcStats subscriber_stats = 4; -} + message Result { + repeated RtcStats publisher_stats = 1; + repeated RtcStats subscriber_stats = 2; + } + required uint64 async_id = 1; + + oneof message { + string error = 2; + Result result = 3; + } +} // // Options // message VideoEncoding { - uint64 max_bitrate = 1; - double max_framerate = 2; + required uint64 max_bitrate = 1; + required double max_framerate = 2; } message AudioEncoding { - uint64 max_bitrate = 1; + required uint64 max_bitrate = 1; } message TrackPublishOptions { // encodings are optional - VideoEncoding video_encoding = 1; - AudioEncoding audio_encoding = 2; - VideoCodec video_codec = 3; - bool dtx = 4; - bool red = 5; - bool simulcast = 6; - TrackSource source = 7; + optional VideoEncoding video_encoding = 1; + optional AudioEncoding audio_encoding = 2; + optional VideoCodec video_codec = 3; + optional bool dtx = 4; + optional bool red = 5; + optional bool simulcast = 6; + optional TrackSource source = 7; + optional string stream = 8; } enum IceTransportType { @@ -182,8 +275,8 @@ enum ContinualGatheringPolicy { message IceServer { repeated string urls = 1; - string username = 2; - string password = 3; + optional string username = 2; + optional string password = 3; } message RtcConfig { @@ -193,12 +286,12 @@ message RtcConfig { } message RoomOptions { - bool auto_subscribe = 1; - bool adaptive_stream = 2; - bool dynacast = 3; + optional bool auto_subscribe = 1; + optional bool adaptive_stream = 2; + optional bool dynacast = 3; optional E2eeOptions e2ee = 4; optional RtcConfig rtc_config = 5; // allow to setup a custom RtcConfiguration - uint32 join_retries = 6; + optional uint32 join_retries = 6; } // @@ -223,163 +316,237 @@ enum DataPacketKind { KIND_RELIABLE = 1; } +enum DisconnectReason { + UNKNOWN_REASON = 0; + // the client initiated the disconnect + CLIENT_INITIATED = 1; + // another participant with the same identity has joined the room + DUPLICATE_IDENTITY = 2; + // the server instance is shutting down + SERVER_SHUTDOWN = 3; + // RoomService.RemoveParticipant was called + PARTICIPANT_REMOVED = 4; + // RoomService.DeleteRoom was called + ROOM_DELETED = 5; + // the client is attempting to resume a session, but server is not aware of it + STATE_MISMATCH = 6; + // client was unable to connect fully + JOIN_FAILURE = 7; + // Cloud-only, the server requested Participant to migrate the connection elsewhere + MIGRATION = 8; + // the signal websocket was closed unexpectedly + SIGNAL_CLOSE = 9; + // the room was closed, due to all Standard and Ingress participants having left + ROOM_CLOSED = 10; +} + +message TranscriptionSegment { + required string id = 1; + required string text = 2; + required uint64 start_time = 3; + required uint64 end_time = 4; + required bool final = 5; + required string language = 6; +} + message BufferInfo { - uint64 data_ptr = 1; - uint64 data_len = 2; + required uint64 data_ptr = 1; + required uint64 data_len = 2; } message OwnedBuffer { - FfiOwnedHandle handle = 1; - BufferInfo data = 2; + required FfiOwnedHandle handle = 1; + required BufferInfo data = 2; } message RoomEvent { - uint64 room_handle = 1; + required uint64 room_handle = 1; oneof message { ParticipantConnected participant_connected = 2; ParticipantDisconnected participant_disconnected = 3; LocalTrackPublished local_track_published = 4; LocalTrackUnpublished local_track_unpublished = 5; - TrackPublished track_published = 6; - TrackUnpublished track_unpublished = 7; - TrackSubscribed track_subscribed = 8; - TrackUnsubscribed track_unsubscribed = 9; - TrackSubscriptionFailed track_subscription_failed = 10; - TrackMuted track_muted = 11; - TrackUnmuted track_unmuted = 12; - ActiveSpeakersChanged active_speakers_changed = 13; - RoomMetadataChanged room_metadata_changed = 14; - ParticipantMetadataChanged participant_metadata_changed = 15; - ParticipantNameChanged participant_name_changed = 16; - ConnectionQualityChanged connection_quality_changed = 17; - ConnectionStateChanged connection_state_changed = 19; - // Connected connected = 20; - Disconnected disconnected = 21; - Reconnecting reconnecting = 22; - Reconnected reconnected = 23; - E2eeStateChanged e2ee_state_changed = 24; - RoomEOS eos = 25; // The stream of room events has ended - DataPacketReceived data_packet_received = 26; + LocalTrackSubscribed local_track_subscribed = 6; + TrackPublished track_published = 7; + TrackUnpublished track_unpublished = 8; + TrackSubscribed track_subscribed = 9; + TrackUnsubscribed track_unsubscribed = 10; + TrackSubscriptionFailed track_subscription_failed = 11; + TrackMuted track_muted = 12; + TrackUnmuted track_unmuted = 13; + ActiveSpeakersChanged active_speakers_changed = 14; + RoomMetadataChanged room_metadata_changed = 15; + RoomSidChanged room_sid_changed = 16; + ParticipantMetadataChanged participant_metadata_changed = 17; + ParticipantNameChanged participant_name_changed = 18; + ParticipantAttributesChanged participant_attributes_changed = 19; + ConnectionQualityChanged connection_quality_changed = 20; + ConnectionStateChanged connection_state_changed = 21; + // Connected connected = 21; + Disconnected disconnected = 22; + Reconnecting reconnecting = 23; + Reconnected reconnected = 24; + E2eeStateChanged e2ee_state_changed = 25; + RoomEOS eos = 26; // The stream of room events has ended + DataPacketReceived data_packet_received = 27; + TranscriptionReceived transcription_received = 28; + ChatMessageReceived chat_message = 29; } } message RoomInfo { - string sid = 1; - string name = 2; - string metadata = 3; + optional string sid = 1; + required string name = 2; + required string metadata = 3; } message OwnedRoom { - FfiOwnedHandle handle = 1; - RoomInfo info = 2; + required FfiOwnedHandle handle = 1; + required RoomInfo info = 2; } -message ParticipantConnected { OwnedParticipant info = 1; } +message ParticipantConnected { required OwnedParticipant info = 1; } message ParticipantDisconnected { - string participant_sid = 1; + required string participant_identity = 1; } message LocalTrackPublished { // The TrackPublicationInfo comes from the PublishTrack response // and the FfiClient musts wait for it before firing this event - string track_sid = 1; + required string track_sid = 1; } message LocalTrackUnpublished { - string publication_sid = 1; + required string publication_sid = 1; +} + +message LocalTrackSubscribed { + required string track_sid = 2; } message TrackPublished { - string participant_sid = 1; - OwnedTrackPublication publication = 2; + required string participant_identity = 1; + required OwnedTrackPublication publication = 2; } message TrackUnpublished { - string participant_sid = 1; - string publication_sid = 2; + required string participant_identity = 1; + required string publication_sid = 2; } // Publication isn't needed for subscription events on the FFI // The FFI will retrieve the publication using the Track sid message TrackSubscribed { - string participant_sid = 1; - OwnedTrack track = 2; + required string participant_identity = 1; + required OwnedTrack track = 2; } message TrackUnsubscribed { // The FFI language can dispose/remove the VideoSink here - string participant_sid = 1; - string track_sid = 2; + required string participant_identity = 1; + required string track_sid = 2; } message TrackSubscriptionFailed { - string participant_sid = 1; - string track_sid = 2; - string error = 3; + required string participant_identity = 1; + required string track_sid = 2; + required string error = 3; } message TrackMuted { - string participant_sid = 1; - string track_sid = 2; + required string participant_identity = 1; + required string track_sid = 2; } message TrackUnmuted { - string participant_sid = 1; - string track_sid = 2; + required string participant_identity = 1; + required string track_sid = 2; } message E2eeStateChanged { - string participant_sid = 1; // Using sid instead of identity for ffi communication - EncryptionState state = 2; + required string participant_identity = 1; // Using sid instead of identity for ffi communication + required EncryptionState state = 2; } -message ActiveSpeakersChanged { repeated string participant_sids = 1; } +message ActiveSpeakersChanged { repeated string participant_identities = 1; } message RoomMetadataChanged { - string metadata = 1; + required string metadata = 1; +} + +message RoomSidChanged { + required string sid = 1; } message ParticipantMetadataChanged { - string participant_sid = 1; - string metadata = 2; + required string participant_identity = 1; + required string metadata = 2; +} + +message ParticipantAttributesChanged { + required string participant_identity = 1; + repeated AttributesEntry attributes = 2; + repeated AttributesEntry changed_attributes = 3; } message ParticipantNameChanged { - string participant_sid = 1; - string name = 2; + required string participant_identity = 1; + required string name = 2; } message ConnectionQualityChanged { - string participant_sid = 1; - ConnectionQuality quality = 2; + required string participant_identity = 1; + required ConnectionQuality quality = 2; } message UserPacket { - OwnedBuffer data = 1; + required OwnedBuffer data = 1; optional string topic = 2; } +message ChatMessage { + required string id = 1; + required int64 timestamp = 2; + required string message = 3; + optional int64 edit_timestamp = 4; + optional bool deleted = 5; + optional bool generated = 6; +} + +message ChatMessageReceived { + required ChatMessage message = 1; + required string participant_identity = 2; +} + message SipDTMF { - uint32 code = 1; + required uint32 code = 1; optional string digit = 2; } message DataPacketReceived { - DataPacketKind kind = 1; - string participant_identity = 2; // Can be empty if the data is sent a server SDK - optional string participant_sid = 3 [deprecated=true]; // Can be empty if the data is sent a server SDK + required DataPacketKind kind = 1; + required string participant_identity = 2; // Can be empty if the data is sent a server SDK oneof value { UserPacket user = 4; SipDTMF sip_dtmf = 5; } } -message ConnectionStateChanged { ConnectionState state = 1; } +message TranscriptionReceived { + optional string participant_identity = 1; + optional string track_sid = 2; + repeated TranscriptionSegment segments = 3; +} + +message ConnectionStateChanged { required ConnectionState state = 1; } message Connected {} -message Disconnected {} +message Disconnected { + required DisconnectReason reason = 1; +} message Reconnecting {} message Reconnected {} message RoomEOS {} + diff --git a/src/protocols/rpc.proto b/src/protocols/rpc.proto new file mode 100644 index 000000000..19fd75f11 --- /dev/null +++ b/src/protocols/rpc.proto @@ -0,0 +1,81 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto2"; + +package livekit.proto; +option csharp_namespace = "LiveKit.Proto"; + +message RpcError { + required uint32 code = 1; + required string message = 2; + optional string data = 3; +} + +// FFI Requests +message PerformRpcRequest { + required uint64 local_participant_handle = 1; + required string destination_identity = 2; + required string method = 3; + required string payload = 4; + optional uint32 response_timeout_ms = 5; +} + +message RegisterRpcMethodRequest { + required uint64 local_participant_handle = 1; + required string method = 2; +} + +message UnregisterRpcMethodRequest { + required uint64 local_participant_handle = 1; + required string method = 2; +} + +message RpcMethodInvocationResponseRequest { + required uint64 local_participant_handle = 1; + required uint64 invocation_id = 2; + optional string payload = 3; + optional RpcError error = 4; +} + +// FFI Responses +message PerformRpcResponse { + required uint64 async_id = 1; +} + +message RegisterRpcMethodResponse {} + +message UnregisterRpcMethodResponse {} + +message RpcMethodInvocationResponseResponse { + optional string error = 1; +} + +// FFI Callbacks +message PerformRpcCallback { + required uint64 async_id = 1; + optional string payload = 2; + optional RpcError error = 3; +} + +// FFI Events +message RpcMethodInvocationEvent { + required uint64 local_participant_handle = 1; + required uint64 invocation_id = 2; + required string method = 3; + required string request_id = 4; + required string caller_identity = 5; + required string payload = 6; + required uint32 response_timeout_ms = 7; +} diff --git a/src/protocols/stats.proto b/src/protocols/stats.proto index 45df5c9b2..f4771f3e1 100644 --- a/src/protocols/stats.proto +++ b/src/protocols/stats.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -91,83 +91,83 @@ enum IceTcpCandidateType { message RtcStats { message Codec { - RtcStatsData rtc = 1; - CodecStats codec = 2; + required RtcStatsData rtc = 1; + required CodecStats codec = 2; } message InboundRtp { - RtcStatsData rtc = 1; - RtpStreamStats stream = 2; - ReceivedRtpStreamStats received = 3; - InboundRtpStreamStats inbound = 4; + required RtcStatsData rtc = 1; + required RtpStreamStats stream = 2; + required ReceivedRtpStreamStats received = 3; + required InboundRtpStreamStats inbound = 4; } message OutboundRtp { - RtcStatsData rtc = 1; - RtpStreamStats stream = 2; - SentRtpStreamStats sent = 3; - OutboundRtpStreamStats outbound = 4; + required RtcStatsData rtc = 1; + required RtpStreamStats stream = 2; + required SentRtpStreamStats sent = 3; + required OutboundRtpStreamStats outbound = 4; } message RemoteInboundRtp { - RtcStatsData rtc = 1; - RtpStreamStats stream = 2; - ReceivedRtpStreamStats received = 3; - RemoteInboundRtpStreamStats remote_inbound = 4; + required RtcStatsData rtc = 1; + required RtpStreamStats stream = 2; + required ReceivedRtpStreamStats received = 3; + required RemoteInboundRtpStreamStats remote_inbound = 4; } message RemoteOutboundRtp { - RtcStatsData rtc = 1; - RtpStreamStats stream = 2; - SentRtpStreamStats sent = 3; - RemoteOutboundRtpStreamStats remote_outbound = 4; + required RtcStatsData rtc = 1; + required RtpStreamStats stream = 2; + required SentRtpStreamStats sent = 3; + required RemoteOutboundRtpStreamStats remote_outbound = 4; } message MediaSource { - RtcStatsData rtc = 1; - MediaSourceStats source = 2; - AudioSourceStats audio = 3; - VideoSourceStats video = 4; + required RtcStatsData rtc = 1; + required MediaSourceStats source = 2; + required AudioSourceStats audio = 3; + required VideoSourceStats video = 4; } message MediaPlayout { - RtcStatsData rtc = 1; - AudioPlayoutStats audio_playout = 2; + required RtcStatsData rtc = 1; + required AudioPlayoutStats audio_playout = 2; } message PeerConnection { - RtcStatsData rtc = 1; - PeerConnectionStats pc = 2; + required RtcStatsData rtc = 1; + required PeerConnectionStats pc = 2; } message DataChannel { - RtcStatsData rtc = 1; - DataChannelStats dc = 2; + required RtcStatsData rtc = 1; + required DataChannelStats dc = 2; } message Transport { - RtcStatsData rtc = 1; - TransportStats transport = 2; + required RtcStatsData rtc = 1; + required TransportStats transport = 2; } message CandidatePair { - RtcStatsData rtc = 1; - CandidatePairStats candidate_pair = 2; + required RtcStatsData rtc = 1; + required CandidatePairStats candidate_pair = 2; } message LocalCandidate { - RtcStatsData rtc = 1; - IceCandidateStats candidate = 2; + required RtcStatsData rtc = 1; + required IceCandidateStats candidate = 2; } message RemoteCandidate { - RtcStatsData rtc = 1; - IceCandidateStats candidate = 2; + required RtcStatsData rtc = 1; + required IceCandidateStats candidate = 2; } message Certificate { - RtcStatsData rtc = 1; - CertificateStats certificate = 2; + required RtcStatsData rtc = 1; + required CertificateStats certificate = 2; } message Track { @@ -194,256 +194,256 @@ message RtcStats { } message RtcStatsData { - string id = 1; - int64 timestamp = 2; + required string id = 1; + required int64 timestamp = 2; } message CodecStats { - uint32 payload_type = 1; - string transport_id = 2; - string mime_type = 3; - uint32 clock_rate = 4; - uint32 channels = 5; - string sdp_fmtp_line = 6; + required uint32 payload_type = 1; + required string transport_id = 2; + required string mime_type = 3; + required uint32 clock_rate = 4; + required uint32 channels = 5; + required string sdp_fmtp_line = 6; } message RtpStreamStats { - uint32 ssrc = 1; - string kind = 2; - string transport_id = 3; - string codec_id = 4; + required uint32 ssrc = 1; + required string kind = 2; + required string transport_id = 3; + required string codec_id = 4; } message ReceivedRtpStreamStats { - uint64 packets_received = 1; - int64 packets_lost = 2; - double jitter = 3; + required uint64 packets_received = 1; + required int64 packets_lost = 2; + required double jitter = 3; } message InboundRtpStreamStats { - string track_identifier = 1; - string mid = 2; - string remote_id = 3; - uint32 frames_decoded = 4; - uint32 key_frames_decoded = 5; - uint32 frames_rendered = 6; - uint32 frames_dropped = 7; - uint32 frame_width = 8; - uint32 frame_height = 9; - double frames_per_second = 10; - uint64 qp_sum = 11; - double total_decode_time = 12; - double total_inter_frame_delay = 13; - double total_squared_inter_frame_delay = 14; - uint32 pause_count = 15; - double total_pause_duration = 16; - uint32 freeze_count = 17; - double total_freeze_duration = 18; - double last_packet_received_timestamp = 19; - uint64 header_bytes_received = 20; - uint64 packets_discarded = 21; - uint64 fec_bytes_received = 22; - uint64 fec_packets_received = 23; - uint64 fec_packets_discarded = 24; - uint64 bytes_received = 25; - uint32 nack_count = 26; - uint32 fir_count = 27; - uint32 pli_count = 28; - double total_processing_delay = 29; - double estimated_playout_timestamp = 30; - double jitter_buffer_delay = 31; - double jitter_buffer_target_delay = 32; - uint64 jitter_buffer_emitted_count = 33; - double jitter_buffer_minimum_delay = 34; - uint64 total_samples_received = 35; - uint64 concealed_samples = 36; - uint64 silent_concealed_samples = 37; - uint64 concealment_events = 38; - uint64 inserted_samples_for_deceleration = 39; - uint64 removed_samples_for_acceleration = 40; - double audio_level = 41; - double total_audio_energy = 42; - double total_samples_duration = 43; - uint64 frames_received = 44; - string decoder_implementation = 45; - string playout_id = 46; - bool power_efficient_decoder = 47; - uint64 frames_assembled_from_multiple_packets = 48; - double total_assembly_time = 49; - uint64 retransmitted_packets_received = 50; - uint64 retransmitted_bytes_received = 51; - uint32 rtx_ssrc = 52; - uint32 fec_ssrc = 53; + required string track_identifier = 1; + required string mid = 2; + required string remote_id = 3; + required uint32 frames_decoded = 4; + required uint32 key_frames_decoded = 5; + required uint32 frames_rendered = 6; + required uint32 frames_dropped = 7; + required uint32 frame_width = 8; + required uint32 frame_height = 9; + required double frames_per_second = 10; + required uint64 qp_sum = 11; + required double total_decode_time = 12; + required double total_inter_frame_delay = 13; + required double total_squared_inter_frame_delay = 14; + required uint32 pause_count = 15; + required double total_pause_duration = 16; + required uint32 freeze_count = 17; + required double total_freeze_duration = 18; + required double last_packet_received_timestamp = 19; + required uint64 header_bytes_received = 20; + required uint64 packets_discarded = 21; + required uint64 fec_bytes_received = 22; + required uint64 fec_packets_received = 23; + required uint64 fec_packets_discarded = 24; + required uint64 bytes_received = 25; + required uint32 nack_count = 26; + required uint32 fir_count = 27; + required uint32 pli_count = 28; + required double total_processing_delay = 29; + required double estimated_playout_timestamp = 30; + required double jitter_buffer_delay = 31; + required double jitter_buffer_target_delay = 32; + required uint64 jitter_buffer_emitted_count = 33; + required double jitter_buffer_minimum_delay = 34; + required uint64 total_samples_received = 35; + required uint64 concealed_samples = 36; + required uint64 silent_concealed_samples = 37; + required uint64 concealment_events = 38; + required uint64 inserted_samples_for_deceleration = 39; + required uint64 removed_samples_for_acceleration = 40; + required double audio_level = 41; + required double total_audio_energy = 42; + required double total_samples_duration = 43; + required uint64 frames_received = 44; + required string decoder_implementation = 45; + required string playout_id = 46; + required bool power_efficient_decoder = 47; + required uint64 frames_assembled_from_multiple_packets = 48; + required double total_assembly_time = 49; + required uint64 retransmitted_packets_received = 50; + required uint64 retransmitted_bytes_received = 51; + required uint32 rtx_ssrc = 52; + required uint32 fec_ssrc = 53; } message SentRtpStreamStats { - uint64 packets_sent = 1; - uint64 bytes_sent = 2; + required uint64 packets_sent = 1; + required uint64 bytes_sent = 2; } message OutboundRtpStreamStats { - string mid = 1; - string media_source_id = 2; - string remote_id = 3; - string rid = 4; - uint64 header_bytes_sent = 5; - uint64 retransmitted_packets_sent = 6; - uint64 retransmitted_bytes_sent = 7; - uint32 rtx_ssrc = 8; - double target_bitrate = 9; - uint64 total_encoded_bytes_target = 10; - uint32 frame_width = 11; - uint32 frame_height = 12; - double frames_per_second = 13; - uint32 frames_sent = 14; - uint32 huge_frames_sent = 15; - uint32 frames_encoded = 16; - uint32 key_frames_encoded = 17; - uint64 qp_sum = 18; - double total_encode_time = 19; - double total_packet_send_delay = 20; - QualityLimitationReason quality_limitation_reason = 21; + required string mid = 1; + required string media_source_id = 2; + required string remote_id = 3; + required string rid = 4; + required uint64 header_bytes_sent = 5; + required uint64 retransmitted_packets_sent = 6; + required uint64 retransmitted_bytes_sent = 7; + required uint32 rtx_ssrc = 8; + required double target_bitrate = 9; + required uint64 total_encoded_bytes_target = 10; + required uint32 frame_width = 11; + required uint32 frame_height = 12; + required double frames_per_second = 13; + required uint32 frames_sent = 14; + required uint32 huge_frames_sent = 15; + required uint32 frames_encoded = 16; + required uint32 key_frames_encoded = 17; + required uint64 qp_sum = 18; + required double total_encode_time = 19; + required double total_packet_send_delay = 20; + required QualityLimitationReason quality_limitation_reason = 21; map quality_limitation_durations = 22; - uint32 quality_limitation_resolution_changes = 23; - uint32 nack_count = 24; - uint32 fir_count = 25; - uint32 pli_count = 26; - string encoder_implementation = 27; - bool power_efficient_encoder = 28; - bool active = 29; - string scalibility_mode = 30; + required uint32 quality_limitation_resolution_changes = 23; + required uint32 nack_count = 24; + required uint32 fir_count = 25; + required uint32 pli_count = 26; + required string encoder_implementation = 27; + required bool power_efficient_encoder = 28; + required bool active = 29; + required string scalability_mode = 30; } message RemoteInboundRtpStreamStats { - string local_id = 1; - double round_trip_time = 2; - double total_round_trip_time = 3; - double fraction_lost = 4; - uint64 round_trip_time_measurements = 5; + required string local_id = 1; + required double round_trip_time = 2; + required double total_round_trip_time = 3; + required double fraction_lost = 4; + required uint64 round_trip_time_measurements = 5; } message RemoteOutboundRtpStreamStats { - string local_id = 1; - double remote_timestamp = 2; - uint64 reports_sent = 3; - double round_trip_time = 4; - double total_round_trip_time = 5; - uint64 round_trip_time_measurements = 6; + required string local_id = 1; + required double remote_timestamp = 2; + required uint64 reports_sent = 3; + required double round_trip_time = 4; + required double total_round_trip_time = 5; + required uint64 round_trip_time_measurements = 6; } message MediaSourceStats { - string track_identifier = 1; - string kind = 2; + required string track_identifier = 1; + required string kind = 2; } message AudioSourceStats { - double audio_level = 1; - double total_audio_energy = 2; - double total_samples_duration = 3; - double echo_return_loss = 4; - double echo_return_loss_enhancement = 5; - double dropped_samples_duration = 6; - uint32 dropped_samples_events = 7; - double total_capture_delay = 8; - uint64 total_samples_captured = 9; + required double audio_level = 1; + required double total_audio_energy = 2; + required double total_samples_duration = 3; + required double echo_return_loss = 4; + required double echo_return_loss_enhancement = 5; + required double dropped_samples_duration = 6; + required uint32 dropped_samples_events = 7; + required double total_capture_delay = 8; + required uint64 total_samples_captured = 9; } message VideoSourceStats { - uint32 width = 1; - uint32 height = 2; - uint32 frames = 3; - double frames_per_second = 4; + required uint32 width = 1; + required uint32 height = 2; + required uint32 frames = 3; + required double frames_per_second = 4; } message AudioPlayoutStats { - string kind = 1; - double synthesized_samples_duration = 2; - uint32 synthesized_samples_events = 3; - double total_samples_duration = 4; - double total_playout_delay = 5; - uint64 total_samples_count = 6; + required string kind = 1; + required double synthesized_samples_duration = 2; + required uint32 synthesized_samples_events = 3; + required double total_samples_duration = 4; + required double total_playout_delay = 5; + required uint64 total_samples_count = 6; } message PeerConnectionStats { - uint32 data_channels_opened = 1; - uint32 data_channels_closed = 2; + required uint32 data_channels_opened = 1; + required uint32 data_channels_closed = 2; } message DataChannelStats { - string label = 1; - string protocol = 2; - int32 data_channel_identifier = 3; + required string label = 1; + required string protocol = 2; + required int32 data_channel_identifier = 3; optional DataChannelState state = 4; - uint32 messages_sent = 5; - uint64 bytes_sent = 6; - uint32 messages_received = 7; - uint64 bytes_received = 8; + required uint32 messages_sent = 5; + required uint64 bytes_sent = 6; + required uint32 messages_received = 7; + required uint64 bytes_received = 8; } message TransportStats { - uint64 packets_sent = 1; - uint64 packets_received = 2; - uint64 bytes_sent = 3; - uint64 bytes_received = 4; - IceRole ice_role = 5; - string ice_local_username_fragment = 6; + required uint64 packets_sent = 1; + required uint64 packets_received = 2; + required uint64 bytes_sent = 3; + required uint64 bytes_received = 4; + required IceRole ice_role = 5; + required string ice_local_username_fragment = 6; optional DtlsTransportState dtls_state = 7; optional IceTransportState ice_state = 8; - string selected_candidate_pair_id = 9; - string local_certificate_id = 10; - string remote_certificate_id = 11; - string tls_version = 12; - string dtls_cipher = 13; - DtlsRole dtls_role = 14; - string srtp_cipher = 15; - uint32 selected_candidate_pair_changes = 16; + required string selected_candidate_pair_id = 9; + required string local_certificate_id = 10; + required string remote_certificate_id = 11; + required string tls_version = 12; + required string dtls_cipher = 13; + required DtlsRole dtls_role = 14; + required string srtp_cipher = 15; + required uint32 selected_candidate_pair_changes = 16; } message CandidatePairStats { - string transport_id = 1; - string local_candidate_id = 2; - string remote_candidate_id = 3; + required string transport_id = 1; + required string local_candidate_id = 2; + required string remote_candidate_id = 3; optional IceCandidatePairState state = 4; - bool nominated = 5; - uint64 packets_sent = 6; - uint64 packets_received = 7; - uint64 bytes_sent = 8; - uint64 bytes_received = 9; - double last_packet_sent_timestamp = 10; - double last_packet_received_timestamp = 11; - double total_round_trip_time = 12; - double current_round_trip_time = 13; - double available_outgoing_bitrate = 14; - double available_incoming_bitrate = 15; - uint64 requests_received = 16; - uint64 requests_sent = 17; - uint64 responses_received = 18; - uint64 responses_sent = 19; - uint64 consent_requests_sent = 20; - uint32 packets_discarded_on_send = 21; - uint64 bytes_discarded_on_send = 22; + required bool nominated = 5; + required uint64 packets_sent = 6; + required uint64 packets_received = 7; + required uint64 bytes_sent = 8; + required uint64 bytes_received = 9; + required double last_packet_sent_timestamp = 10; + required double last_packet_received_timestamp = 11; + required double total_round_trip_time = 12; + required double current_round_trip_time = 13; + required double available_outgoing_bitrate = 14; + required double available_incoming_bitrate = 15; + required uint64 requests_received = 16; + required uint64 requests_sent = 17; + required uint64 responses_received = 18; + required uint64 responses_sent = 19; + required uint64 consent_requests_sent = 20; + required uint32 packets_discarded_on_send = 21; + required uint64 bytes_discarded_on_send = 22; } message IceCandidateStats { - string transport_id = 1; - string address = 2; - int32 port = 3; - string protocol = 4; + required string transport_id = 1; + required string address = 2; + required int32 port = 3; + required string protocol = 4; optional IceCandidateType candidate_type = 5; - int32 priority = 6; - string url = 7; + required int32 priority = 6; + required string url = 7; optional IceServerTransportProtocol relay_protocol = 8; - string foundation = 9; - string related_address = 10; - int32 related_port = 11; - string username_fragment = 12; + required string foundation = 9; + required string related_address = 10; + required int32 related_port = 11; + required string username_fragment = 12; optional IceTcpCandidateType tcp_type = 13; } message CertificateStats { - string fingerprint = 1; - string fingerprint_algorithm = 2; - string base64_certificate = 3; - string issuer_certificate_id = 4; + required string fingerprint = 1; + required string fingerprint_algorithm = 2; + required string base64_certificate = 3; + required string issuer_certificate_id = 4; } diff --git a/src/protocols/track.proto b/src/protocols/track.proto index 38fb799d7..4bebd399c 100644 --- a/src/protocols/track.proto +++ b/src/protocols/track.proto @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; @@ -23,30 +23,30 @@ import "stats.proto"; // Create a new VideoTrack from a VideoSource message CreateVideoTrackRequest { - string name = 1; - uint64 source_handle = 2; + required string name = 1; + required uint64 source_handle = 2; } message CreateVideoTrackResponse { - OwnedTrack track = 1; + required OwnedTrack track = 1; } // Create a new AudioTrack from a AudioSource message CreateAudioTrackRequest { - string name = 1; - uint64 source_handle = 2; + required string name = 1; + required uint64 source_handle = 2; } message CreateAudioTrackResponse { - OwnedTrack track = 1; + required OwnedTrack track = 1; } message GetStatsRequest { - uint64 track_handle = 1; + required uint64 track_handle = 1; } message GetStatsResponse { - uint64 async_id = 1; + required uint64 async_id = 1; } message GetStatsCallback { - uint64 async_id = 1; + required uint64 async_id = 1; optional string error = 2; repeated RtcStats stats = 3; } @@ -78,34 +78,54 @@ enum StreamState { } message TrackPublicationInfo { - string sid = 1; - string name = 2; - TrackKind kind = 3; - TrackSource source = 4; - bool simulcasted = 5; - uint32 width = 6; - uint32 height = 7; - string mime_type = 8; - bool muted = 9; - bool remote = 10; - EncryptionType encryption_type = 11; + required string sid = 1; + required string name = 2; + required TrackKind kind = 3; + required TrackSource source = 4; + required bool simulcasted = 5; + required uint32 width = 6; + required uint32 height = 7; + required string mime_type = 8; + required bool muted = 9; + required bool remote = 10; + required EncryptionType encryption_type = 11; } message OwnedTrackPublication { - FfiOwnedHandle handle = 1; - TrackPublicationInfo info = 2; + required FfiOwnedHandle handle = 1; + required TrackPublicationInfo info = 2; } message TrackInfo { - string sid = 1; - string name = 2; - TrackKind kind = 3; - StreamState stream_state = 4; - bool muted = 5; - bool remote = 6; + required string sid = 1; + required string name = 2; + required TrackKind kind = 3; + required StreamState stream_state = 4; + required bool muted = 5; + required bool remote = 6; } message OwnedTrack { - FfiOwnedHandle handle = 1; - TrackInfo info = 2; + required FfiOwnedHandle handle = 1; + required TrackInfo info = 2; +} + +// Mute/UnMute a track +message LocalTrackMuteRequest { + required uint64 track_handle = 1; + required bool mute = 2; +} + +message LocalTrackMuteResponse { + required bool muted = 1; +} + +// Enable/Disable a remote track +message EnableRemoteTrackRequest { + required uint64 track_handle = 1; + required bool enabled = 2; +} + +message EnableRemoteTrackResponse { + required bool enabled = 1; } diff --git a/src/protocols/video_frame.proto b/src/protocols/video_frame.proto index ffb2e0bde..6ce0a2bc8 100644 --- a/src/protocols/video_frame.proto +++ b/src/protocols/video_frame.proto @@ -12,52 +12,66 @@ // See the License for the specific language governing permissions and // limitations under the License. -syntax = "proto3"; +syntax = "proto2"; package livekit.proto; option csharp_namespace = "LiveKit.Proto"; import "handle.proto"; +import "track.proto"; // Create a new VideoStream // VideoStream is used to receive video frames from a track message NewVideoStreamRequest { - uint64 track_handle = 1; - VideoStreamType type = 2; + required uint64 track_handle = 1; + required VideoStreamType type = 2; // Get the frame on a specific format optional VideoBufferType format = 3; - bool normalize_stride = 4; // if true, stride will be set to width/chroma_width + optional bool normalize_stride = 4; // if true, stride will be set to width/chroma_width } -message NewVideoStreamResponse { OwnedVideoStream stream = 1; } +message NewVideoStreamResponse { required OwnedVideoStream stream = 1; } + +// Request a video stream from a participant +message VideoStreamFromParticipantRequest { + required uint64 participant_handle = 1; + required VideoStreamType type = 2; + required TrackSource track_source = 3; + optional VideoBufferType format = 4; + optional bool normalize_stride = 5; +} + +message VideoStreamFromParticipantResponse { required OwnedVideoStream stream = 1;} // Create a new VideoSource // VideoSource is used to send video frame to a track message NewVideoSourceRequest { - VideoSourceType type = 1; + required VideoSourceType type = 1; // Used to determine which encodings to use + simulcast layers // Most of the time it corresponds to the source resolution - VideoSourceResolution resolution = 2; + required VideoSourceResolution resolution = 2; } -message NewVideoSourceResponse { OwnedVideoSource source = 1; } +message NewVideoSourceResponse { required OwnedVideoSource source = 1; } // Push a frame to a VideoSource message CaptureVideoFrameRequest { - uint64 source_handle = 1; - VideoBufferInfo buffer = 2; - int64 timestamp_us = 3; // In microseconds - VideoRotation rotation = 4; + required uint64 source_handle = 1; + required VideoBufferInfo buffer = 2; + required int64 timestamp_us = 3; // In microseconds + required VideoRotation rotation = 4; } message CaptureVideoFrameResponse {} message VideoConvertRequest { - bool flip_y = 1; - VideoBufferInfo buffer = 2; - VideoBufferType dst_type = 3; + optional bool flip_y = 1; + required VideoBufferInfo buffer = 2; + required VideoBufferType dst_type = 3; } -message VideoConvertResponse { - optional string error = 1; - OwnedVideoBuffer buffer = 2; +message VideoConvertResponse { + oneof message { + string error = 1; + OwnedVideoBuffer buffer = 2; + } } // @@ -65,9 +79,9 @@ message VideoConvertResponse { // message VideoResolution { - uint32 width = 1; - uint32 height = 2; - double frame_rate = 3; + required uint32 width = 1; + required uint32 height = 2; + required double frame_rate = 3; } enum VideoCodec { @@ -100,21 +114,21 @@ enum VideoBufferType { message VideoBufferInfo { message ComponentInfo { - uint64 data_ptr = 1; - uint32 stride = 2; - uint32 size = 3; + required uint64 data_ptr = 1; + required uint32 stride = 2; + required uint32 size = 3; } - VideoBufferType type = 1; - uint32 width = 2; - uint32 height = 3; - uint64 data_ptr = 4; - uint32 stride = 6; // only for packed formats + required VideoBufferType type = 1; + required uint32 width = 2; + required uint32 height = 3; + required uint64 data_ptr = 4; + required uint32 stride = 6; // only for packed formats repeated ComponentInfo components = 7; } message OwnedVideoBuffer { - FfiOwnedHandle handle = 1; - VideoBufferInfo info = 2; + required FfiOwnedHandle handle = 1; + required VideoBufferInfo info = 2; } // @@ -128,16 +142,16 @@ enum VideoStreamType { } message VideoStreamInfo { - VideoStreamType type = 1; + required VideoStreamType type = 1; } message OwnedVideoStream { - FfiOwnedHandle handle = 1; - VideoStreamInfo info = 2; + required FfiOwnedHandle handle = 1; + required VideoStreamInfo info = 2; } message VideoStreamEvent { - uint64 stream_handle = 1; + required uint64 stream_handle = 1; oneof message { VideoFrameReceived frame_received = 2; VideoStreamEOS eos = 3; @@ -145,9 +159,9 @@ message VideoStreamEvent { } message VideoFrameReceived { - OwnedVideoBuffer buffer = 1; - int64 timestamp_us = 2; // In microseconds - VideoRotation rotation = 3; + required OwnedVideoBuffer buffer = 1; + required int64 timestamp_us = 2; // In microseconds + required VideoRotation rotation = 3; } message VideoStreamEOS {} @@ -157,8 +171,8 @@ message VideoStreamEOS {} // message VideoSourceResolution { - uint32 width = 1; - uint32 height = 2; + required uint32 width = 1; + required uint32 height = 2; } enum VideoSourceType { @@ -166,10 +180,10 @@ enum VideoSourceType { } message VideoSourceInfo { - VideoSourceType type = 1; + required VideoSourceType type = 1; } message OwnedVideoSource { - FfiOwnedHandle handle = 1; - VideoSourceInfo info = 2; + required FfiOwnedHandle handle = 1; + required VideoSourceInfo info = 2; } diff --git a/src/qml/CallPage.qml b/src/qml/CallPage.qml index 987420f09..0033425d7 100644 --- a/src/qml/CallPage.qml +++ b/src/qml/CallPage.qml @@ -12,66 +12,46 @@ Kirigami.Page { title: i18nc("@title", "Call") - VideoOutput { - id: video + RowLayout { anchors.fill: parent - visible: false - Component.onCompleted: CallController.setVideoSink(video.videoSink) - } - VideoOutput { - id: viewFinder - anchors.centerIn: parent + VideoOutput { + id: viewFinder - ToolBar { - id: toolbar + Layout.fillWidth: parent.width / 2 + Layout.preferredHeight: parent.height - anchors.horizontalCenter: parent.horizontalCenter - anchors.bottom: parent.bottom - anchors.bottomMargin: Kirigami.Units.gridUnit * 8 - - z: 1000 - - background: Kirigami.ShadowedRectangle { - color: Kirigami.Theme.backgroundColor - radius: 5 - - shadow { - size: 15 - yOffset: 3 - color: Qt.rgba(0, 0, 0, 0.2) - } - - border { - color: Kirigami.ColorUtils.tintWithAlpha(Kirigami.Theme.backgroundColor, Kirigami.Theme.textColor, 0.2) - width: 1 - } - - Kirigami.Theme.inherit: false - Kirigami.Theme.colorSet: Kirigami.Theme.Window + Label { + text: "You" } - RowLayout { - ToolButton { - id: cameraButton - icon.name: "camera-on-symbolic" - text: i18nc("@action:button", "Enable Camera") - display: AbstractButton.IconOnly - checkable: true - onClicked: CallController.toggleCamera() + } - ToolTip.text: text - ToolTip.visible: hovered - ToolTip.delay: Kirigami.Units.toolTipDelay - } + VideoOutput { + id: otherViewFinder + + Layout.fillWidth: parent.width / 2 + Layout.preferredHeight: parent.height + + Label { + text: "Them" } } } LivekitVideoSink { - videoSink: viewFinder.videoSink + videoSink: otherViewFinder.videoSink } - //Component.onCompleted: camera.start() + Component.onCompleted: camera.start() + + Connections { + target: CallController + + function onConnected(): void { + CallController.setCameraVideoSink(viewFinder.videoSink) + CallController.toggleCamera() + } + } CaptureSession { camera: Camera { @@ -82,5 +62,4 @@ Kirigami.Page { } videoOutput: viewFinder } - }