ESCA111/neochat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to latest LiveKit, fix up QML

Browse Source
This commit is contained in:
Joshua Goins
2024-11-22 08:53:03 -05:00
committed by Tobias Fella
parent ddc16a17d2
commit 2f39e70b67
14 changed files with 1053 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/CMakeLists.txt
View File

@@ -411,10 +411,6 @@ if (NOT ANDROID AND NOT WIN32 AND NOT APPLE AND NOT HAIKU)
target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER) target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER)
target_compile_definitions(neochat PUBLIC -DHAVE_X11=1) target_compile_definitions(neochat PUBLIC -DHAVE_X11=1)
target_sources(neochat PRIVATE runner.cpp) target_sources(neochat PRIVATE runner.cpp)
if (NOT ANDROID AND NOT WIN32 AND NOT APPLE)
#target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER)
target_compile_definitions(neochat PUBLIC -DHAVE_X11)
#target_sources(neochat PRIVATE runner.cpp)
if (TARGET KUnifiedPush) if (TARGET KUnifiedPush)
target_sources(neochat PRIVATE fakerunner.cpp) target_sources(neochat PRIVATE fakerunner.cpp)
@@ -435,9 +431,10 @@ qt_add_protobuf(neochat
protocols/participant.proto protocols/participant.proto
protocols/stats.proto protocols/stats.proto
protocols/track.proto protocols/track.proto
protocols/rpc.proto
) )
target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums) target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums ${CMAKE_CURRENT_SOURCE_DIR}/calls)
target_link_libraries(neochat PRIVATE settingsplugin timelineplugin devtoolsplugin loginplugin chatbarplugin) target_link_libraries(neochat PRIVATE settingsplugin timelineplugin devtoolsplugin loginplugin chatbarplugin)
target_link_libraries(neochat PUBLIC target_link_libraries(neochat PUBLIC
Qt::Core Qt::Core

32
src/calls/callcontroller.cpp
View File

@@ -11,7 +11,6 @@
#include <QVideoFrame> #include <QVideoFrame>
#include <QVideoFrameFormat> #include <QVideoFrameFormat>
#include <QVideoSink> #include <QVideoSink>
#include <qprotobufregistration.h>
#include <livekit_ffi.h> #include <livekit_ffi.h>
@@ -29,7 +28,7 @@ using namespace livekit::proto;
using namespace Quotient; using namespace Quotient;
extern "C" { extern "C" {
void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs); void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs, const char *, const char *);
} }
void callback(const uint8_t *data, size_t length) void callback(const uint8_t *data, size_t length)
@@ -50,7 +49,7 @@ CallController::CallController()
void CallController::init() void CallController::init()
{ {
qRegisterProtobufTypes(); qRegisterProtobufTypes();
livekit_ffi_initialize(callback, true); livekit_ffi_initialize(callback, true, "test", "1.0");
} }
static void handleLog(LogRecordRepeated &&logs) static void handleLog(LogRecordRepeated &&logs)
@@ -65,15 +64,16 @@ static void handleLog(LogRecordRepeated &&logs)
void CallController::handleConnect(ConnectCallback &&callback) void CallController::handleConnect(ConnectCallback &&callback)
{ {
qWarning() << "Connecting to" << callback.room().info().name() << "with id" << callback.asyncId(); qWarning() << "Connecting to" << callback.result().room().info().name() << "with id" << callback.asyncId();
if (!m_connectingRooms.contains(callback.asyncId()) || !m_connectingRooms[callback.asyncId()] if (!m_connectingRooms.contains(callback.asyncId()) || !m_connectingRooms[callback.asyncId()]
|| m_connectingRooms[callback.asyncId()]->id() != callback.room().info().name()) { || m_connectingRooms[callback.asyncId()]->id() != callback.room().info().name()) {
qWarning() << "Connecting to unexpected room"; qWarning() << "Connecting to unexpected room";
return; return;
} }
m_connectingRooms.remove(callback.asyncId()); m_connectingRooms.remove(callback.asyncId());
m_rooms[callback.asyncId()] = callback.room(); m_rooms[callback.asyncId()] = callback.result().room();
localParticipant = callback.localParticipant().handle().id_proto(); localParticipant = callback.result().localParticipant().handle().id_proto();
Q_EMIT connected();
} }
void CallController::handleDispose(DisposeCallback &&callback) void CallController::handleDispose(DisposeCallback &&callback)
@@ -92,7 +92,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event)
if (event.hasParticipantConnected()) { if (event.hasParticipantConnected()) {
qWarning() << "Participant connected" << event.participantConnected().info().info().identity(); qWarning() << "Participant connected" << event.participantConnected().info().info().identity();
} else if (event.hasParticipantDisconnected()) { } else if (event.hasParticipantDisconnected()) {
qWarning() << "Participant connected" << event.participantDisconnected().participantSid(); qWarning() << "Participant connected" << event.participantDisconnected().participantIdentity();
} else if (event.hasLocalTrackPublished()) { } else if (event.hasLocalTrackPublished()) {
qWarning() << "Local track published"; qWarning() << "Local track published";
m_localVideoTrackSid = event.localTrackPublished().trackSid(); m_localVideoTrackSid = event.localTrackPublished().trackSid();
@@ -140,7 +140,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event)
} else if (event.hasTrackUnmuted()) { } else if (event.hasTrackUnmuted()) {
qWarning() << "Track unmuted"; qWarning() << "Track unmuted";
} else if (event.hasActiveSpeakersChanged()) { } else if (event.hasActiveSpeakersChanged()) {
// qWarning() << "Active speakers changed"; qWarning() << "Active speakers changed";
} else if (event.hasRoomMetadataChanged()) { } else if (event.hasRoomMetadataChanged()) {
qWarning() << "room metadata changed"; qWarning() << "room metadata changed";
} else if (event.hasParticipantMetadataChanged()) { } else if (event.hasParticipantMetadataChanged()) {
@@ -265,10 +265,6 @@ void CallController::handleEvent(FfiEvent &&event)
qWarning() << "publish data"; qWarning() << "publish data";
} else if (event.hasCaptureAudioFrame()) { } else if (event.hasCaptureAudioFrame()) {
qWarning() << "audio frame"; qWarning() << "audio frame";
} else if (event.hasUpdateLocalMetadata()) {
qWarning() << "update local metadata";
} else if (event.hasUpdateLocalName()) {
qWarning() << "update local name";
} else if (event.hasGetStats()) { } else if (event.hasGetStats()) {
qWarning() << "get stats"; qWarning() << "get stats";
} else if (event.hasGetSessionStats()) { } else if (event.hasGetSessionStats()) {
@@ -297,11 +293,11 @@ void CallController::handleCallMemberEvent(const Quotient::CallMemberEvent *even
{"device_id"_ls, connection->deviceId()}, {"device_id"_ls, connection->deviceId()},
}) })
.toJson(); .toJson();
if (event->memberships().isEmpty()) { // This is an old event!
if (!event->contentJson().contains("foci_preferred"_ls)) {
return; return;
} }
auto membership = event->memberships()[0].toObject(); QNetworkRequest request(QUrl((event->contentJson()["foci_preferred"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls)));
QNetworkRequest request(QUrl((membership["foci_active"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls)));
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"_ls); request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"_ls);
auto reply = nam->post(request, json); auto reply = nam->post(request, json);
connect(reply, &QNetworkReply::finished, this, [reply, this, room]() { connect(reply, &QNetworkReply::finished, this, [reply, this, room]() {
@@ -363,6 +359,7 @@ void CallController::setCameraVideoSink(QVideoSink *videoSink)
ffiRequest.setNewVideoSource(newVideoSourceRequest); ffiRequest.setNewVideoSource(newVideoSourceRequest);
auto response = request(std::move(ffiRequest)); auto response = request(std::move(ffiRequest));
handle = response.newVideoSource().source().handle().id_proto(); handle = response.newVideoSource().source().handle().id_proto();
m_localVideoTrackHandle = handle;
CreateVideoTrackRequest createVideoTrackRequest; CreateVideoTrackRequest createVideoTrackRequest;
createVideoTrackRequest.setName("Camera"_ls); createVideoTrackRequest.setName("Camera"_ls);
@@ -410,7 +407,7 @@ void CallController::setVideoSink(QObject *sink)
void LivekitVideoSink::setVideoSink(QVideoSink *videoSink) void LivekitVideoSink::setVideoSink(QVideoSink *videoSink)
{ {
m_videoSink = videoSink; m_videoSink = videoSink;
CallController::instance().setCameraVideoSink(videoSink); CallController::instance().setVideoSink(videoSink);
Q_EMIT videoSinkChanged(); Q_EMIT videoSinkChanged();
} }
QVideoSink *LivekitVideoSink::videoSink() const QVideoSink *LivekitVideoSink::videoSink() const
@@ -437,10 +434,11 @@ void CallController::publishTrack(uint64_t id)
{ {
PublishTrackRequest publishTrackRequest; PublishTrackRequest publishTrackRequest;
publishTrackRequest.setTrackHandle(id); publishTrackRequest.setTrackHandle(m_localVideoTrackHandle);
publishTrackRequest.setLocalParticipantHandle(localParticipant); publishTrackRequest.setLocalParticipantHandle(localParticipant);
TrackPublishOptions options; TrackPublishOptions options;
options.setSource(TrackSourceGadget::SOURCE_CAMERA); options.setSource(TrackSourceGadget::SOURCE_CAMERA);
options.setVideoCodec(VideoCodecGadget::VideoCodec::VP8);
publishTrackRequest.setOptions(options); publishTrackRequest.setOptions(options);
auto request = FfiRequest(); auto request = FfiRequest();

4
src/calls/callcontroller.h
View File

@@ -48,12 +48,13 @@ public:
// Internal. Do not use. // Internal. Do not use.
void handleEvent(livekit::proto::FfiEvent &&event); void handleEvent(livekit::proto::FfiEvent &&event);
Q_INVOKABLE void setVideoSink(QObject *sink); Q_INVOKABLE void setVideoSink(QObject *sink);
void setCameraVideoSink(QVideoSink *videoSink); Q_INVOKABLE void setCameraVideoSink(QVideoSink *videoSink);
Q_INVOKABLE void toggleCamera(); Q_INVOKABLE void toggleCamera();
Q_SIGNALS: Q_SIGNALS:
void callStarted(); void callStarted();
void connected();
private: private:
CallController(); CallController();
@@ -74,6 +75,7 @@ private:
uint64_t localParticipant = 100000; uint64_t localParticipant = 100000;
QString m_localVideoTrackSid; QString m_localVideoTrackSid;
uint64_t m_localVideoTrackId; uint64_t m_localVideoTrackId;
uint64_t m_localVideoTrackHandle;
}; };
class LivekitVideoSink : public QObject class LivekitVideoSink : public QObject

174
src/protocols/audio_frame.proto
View File

@@ -12,76 +12,162 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
import "handle.proto"; import "handle.proto";
import "track.proto";
// Create a new AudioStream // Create a new AudioStream
// AudioStream is used to receive audio frames from a track // AudioStream is used to receive audio frames from a track
message NewAudioStreamRequest { message NewAudioStreamRequest {
uint64 track_handle = 1; required uint64 track_handle = 1;
AudioStreamType type = 2; required AudioStreamType type = 2;
optional uint32 sample_rate = 3;
optional uint32 num_channels = 4;
} }
message NewAudioStreamResponse { OwnedAudioStream stream = 1; } message NewAudioStreamResponse { required OwnedAudioStream stream = 1; }
message AudioStreamFromParticipantRequest {
required uint64 participant_handle = 1;
required AudioStreamType type = 2;
optional TrackSource track_source = 3;
optional uint32 sample_rate = 5;
optional uint32 num_channels = 6;
}
message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; }
// Create a new AudioSource // Create a new AudioSource
message NewAudioSourceRequest { message NewAudioSourceRequest {
AudioSourceType type = 1; required AudioSourceType type = 1;
optional AudioSourceOptions options = 2; optional AudioSourceOptions options = 2;
uint32 sample_rate = 3; required uint32 sample_rate = 3;
uint32 num_channels = 4; required uint32 num_channels = 4;
optional uint32 queue_size_ms = 5;
} }
message NewAudioSourceResponse { OwnedAudioSource source = 1; } message NewAudioSourceResponse { required OwnedAudioSource source = 1; }
// Push a frame to an AudioSource // Push a frame to an AudioSource
// The data provided must be available as long as the client receive the callback. // The data provided must be available as long as the client receive the callback.
message CaptureAudioFrameRequest { message CaptureAudioFrameRequest {
uint64 source_handle = 1; required uint64 source_handle = 1;
AudioFrameBufferInfo buffer = 2; required AudioFrameBufferInfo buffer = 2;
} }
message CaptureAudioFrameResponse { message CaptureAudioFrameResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message CaptureAudioFrameCallback { message CaptureAudioFrameCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2; optional string error = 2;
} }
message ClearAudioBufferRequest {
required uint64 source_handle = 1;
}
message ClearAudioBufferResponse {}
// Create a new AudioResampler // Create a new AudioResampler
message NewAudioResamplerRequest {} message NewAudioResamplerRequest {}
message NewAudioResamplerResponse { message NewAudioResamplerResponse {
OwnedAudioResampler resampler = 1; required OwnedAudioResampler resampler = 1;
} }
// Remix and resample an audio frame // Remix and resample an audio frame
message RemixAndResampleRequest { message RemixAndResampleRequest {
uint64 resampler_handle = 1; required uint64 resampler_handle = 1;
AudioFrameBufferInfo buffer = 2; required AudioFrameBufferInfo buffer = 2;
uint32 num_channels = 3; required uint32 num_channels = 3;
uint32 sample_rate = 4; required uint32 sample_rate = 4;
} }
message RemixAndResampleResponse { message RemixAndResampleResponse {
OwnedAudioFrameBuffer buffer = 1; required OwnedAudioFrameBuffer buffer = 1;
} }
// New resampler using SoX (much better quality)
message NewSoxResamplerRequest {
required double input_rate = 1;
required double output_rate = 2;
required uint32 num_channels = 3;
required SoxResamplerDataType input_data_type = 4;
required SoxResamplerDataType output_data_type = 5;
required SoxQualityRecipe quality_recipe = 6;
optional uint32 flags = 7;
}
message NewSoxResamplerResponse {
oneof message {
OwnedSoxResampler resampler = 1;
string error = 2;
}
}
message PushSoxResamplerRequest {
required uint64 resampler_handle = 1;
required uint64 data_ptr = 2; // *const i16
required uint32 size = 3; // in bytes
}
message PushSoxResamplerResponse {
required uint64 output_ptr = 1; // *const i16 (could be null)
required uint32 size = 2; // in bytes
optional string error = 3;
}
message FlushSoxResamplerRequest {
required uint64 resampler_handle = 1;
}
message FlushSoxResamplerResponse {
required uint64 output_ptr = 1; // *const i16 (could be null)
required uint32 size = 2; // in bytes
optional string error = 3;
}
enum SoxResamplerDataType {
// TODO(theomonnom): support other datatypes (shouldn't really be needed)
SOXR_DATATYPE_INT16I = 0;
SOXR_DATATYPE_INT16S = 1;
}
enum SoxQualityRecipe {
SOXR_QUALITY_QUICK = 0;
SOXR_QUALITY_LOW = 1;
SOXR_QUALITY_MEDIUM = 2;
SOXR_QUALITY_HIGH = 3;
SOXR_QUALITY_VERYHIGH = 4;
}
enum SoxFlagBits {
SOXR_ROLLOFF_SMALL = 0; // 1 << 0
SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
SOXR_ROLLOFF_NONE = 2; // 1 << 2
SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
SOXR_DOUBLE_PRECISION = 4; // 1 << 4
SOXR_VR = 5; // 1 << 5
}
// //
// AudioFrame buffer // AudioFrame buffer
// //
message AudioFrameBufferInfo { message AudioFrameBufferInfo {
uint64 data_ptr = 1; // *const i16 required uint64 data_ptr = 1; // *const i16
uint32 num_channels = 2; required uint32 num_channels = 2;
uint32 sample_rate = 3; required uint32 sample_rate = 3;
uint32 samples_per_channel = 4; required uint32 samples_per_channel = 4;
} }
message OwnedAudioFrameBuffer { message OwnedAudioFrameBuffer {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
AudioFrameBufferInfo info = 2; required AudioFrameBufferInfo info = 2;
} }
// //
@@ -94,16 +180,16 @@ enum AudioStreamType {
} }
message AudioStreamInfo { message AudioStreamInfo {
AudioStreamType type = 1; required AudioStreamType type = 1;
} }
message OwnedAudioStream { message OwnedAudioStream {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
AudioStreamInfo info = 2; required AudioStreamInfo info = 2;
} }
message AudioStreamEvent { message AudioStreamEvent {
uint64 stream_handle = 1; required uint64 stream_handle = 1;
oneof message { oneof message {
AudioFrameReceived frame_received = 2; AudioFrameReceived frame_received = 2;
AudioStreamEOS eos = 3; AudioStreamEOS eos = 3;
@@ -111,7 +197,7 @@ message AudioStreamEvent {
} }
message AudioFrameReceived { message AudioFrameReceived {
OwnedAudioFrameBuffer frame = 1; required OwnedAudioFrameBuffer frame = 1;
} }
message AudioStreamEOS {} message AudioStreamEOS {}
@@ -121,9 +207,9 @@ message AudioStreamEOS {}
// //
message AudioSourceOptions { message AudioSourceOptions {
bool echo_cancellation = 1; required bool echo_cancellation = 1;
bool noise_suppression = 2; required bool noise_suppression = 2;
bool auto_gain_control = 3; required bool auto_gain_control = 3;
} }
enum AudioSourceType { enum AudioSourceType {
@@ -131,12 +217,12 @@ enum AudioSourceType {
} }
message AudioSourceInfo { message AudioSourceInfo {
AudioSourceType type = 2; required AudioSourceType type = 2;
} }
message OwnedAudioSource { message OwnedAudioSource {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
AudioSourceInfo info = 2; required AudioSourceInfo info = 2;
} }
// //
@@ -146,6 +232,20 @@ message OwnedAudioSource {
message AudioResamplerInfo { } message AudioResamplerInfo { }
message OwnedAudioResampler { message OwnedAudioResampler {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
AudioResamplerInfo info = 2; required AudioResamplerInfo info = 2;
}
//
// Sox AudioResampler
//
message SoxResamplerInfo {}
message OwnedSoxResampler {
required FfiOwnedHandle handle = 1;
required SoxResamplerInfo info = 2;
} }

64
src/protocols/e2ee.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -26,23 +26,23 @@ enum EncryptionType {
} }
message FrameCryptor { message FrameCryptor {
string participant_identity = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
int32 key_index = 3; required int32 key_index = 3;
bool enabled = 4; required bool enabled = 4;
} }
message KeyProviderOptions { message KeyProviderOptions {
// Only specify if you want to use a shared_key // Only specify if you want to use a shared_key
optional bytes shared_key = 1; optional bytes shared_key = 1;
int32 ratchet_window_size = 2; required int32 ratchet_window_size = 2;
bytes ratchet_salt = 3; required bytes ratchet_salt = 3;
int32 failure_tolerance = 4; // -1 = no tolerence required int32 failure_tolerance = 4; // -1 = no tolerance
} }
message E2eeOptions { message E2eeOptions {
EncryptionType encryption_type = 1; required EncryptionType encryption_type = 1;
KeyProviderOptions key_provider_options = 2; required KeyProviderOptions key_provider_options = 2;
} }
enum EncryptionState { enum EncryptionState {
@@ -56,7 +56,7 @@ enum EncryptionState {
} }
message E2eeManagerSetEnabledRequest { message E2eeManagerSetEnabledRequest {
bool enabled = 1; required bool enabled = 1;
} }
message E2eeManagerSetEnabledResponse {} message E2eeManagerSetEnabledResponse {}
@@ -66,64 +66,64 @@ message E2eeManagerGetFrameCryptorsResponse {
} }
message FrameCryptorSetEnabledRequest { message FrameCryptorSetEnabledRequest {
string participant_identity = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
bool enabled = 3; required bool enabled = 3;
} }
message FrameCryptorSetEnabledResponse { } message FrameCryptorSetEnabledResponse {}
message FrameCryptorSetKeyIndexRequest { message FrameCryptorSetKeyIndexRequest {
string participant_identity = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
int32 key_index = 3; required int32 key_index = 3;
} }
message FrameCryptorSetKeyIndexResponse { } message FrameCryptorSetKeyIndexResponse {}
message SetSharedKeyRequest { message SetSharedKeyRequest {
bytes shared_key = 1; required bytes shared_key = 1;
int32 key_index = 2; required int32 key_index = 2;
} }
message SetSharedKeyResponse { } message SetSharedKeyResponse {}
message RatchetSharedKeyRequest { message RatchetSharedKeyRequest {
int32 key_index = 1; required int32 key_index = 1;
} }
message RatchetSharedKeyResponse { message RatchetSharedKeyResponse {
optional bytes new_key = 1; optional bytes new_key = 1;
} }
message GetSharedKeyRequest { message GetSharedKeyRequest {
int32 key_index = 1; required int32 key_index = 1;
} }
message GetSharedKeyResponse { message GetSharedKeyResponse {
optional bytes key = 1; optional bytes key = 1;
} }
message SetKeyRequest { message SetKeyRequest {
string participant_identity = 1; required string participant_identity = 1;
bytes key = 2; required bytes key = 2;
int32 key_index = 3; required int32 key_index = 3;
} }
message SetKeyResponse {} message SetKeyResponse {}
message RatchetKeyRequest { message RatchetKeyRequest {
string participant_identity = 1; required string participant_identity = 1;
int32 key_index = 2; required int32 key_index = 2;
} }
message RatchetKeyResponse { message RatchetKeyResponse {
optional bytes new_key = 1; optional bytes new_key = 1;
} }
message GetKeyRequest { message GetKeyRequest {
string participant_identity = 1; required string participant_identity = 1;
int32 key_index = 2; required int32 key_index = 2;
} }
message GetKeyResponse { message GetKeyResponse {
optional bytes key = 1; optional bytes key = 1;
} }
message E2eeRequest { message E2eeRequest {
uint64 room_handle = 1; required uint64 room_handle = 1;
oneof message { oneof message {
E2eeManagerSetEnabledRequest manager_set_enabled = 2; E2eeManagerSetEnabledRequest manager_set_enabled = 2;
E2eeManagerGetFrameCryptorsRequest manager_get_frame_cryptors = 3; E2eeManagerGetFrameCryptorsRequest manager_get_frame_cryptors = 3;

148
src/protocols/ffi.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -23,6 +23,7 @@ import "track.proto";
import "room.proto"; import "room.proto";
import "video_frame.proto"; import "video_frame.proto";
import "audio_frame.proto"; import "audio_frame.proto";
import "rpc.proto";
// **How is the livekit-ffi working: // **How is the livekit-ffi working:
// We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we // We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we
@@ -63,28 +64,48 @@ message FfiRequest {
UnpublishTrackRequest unpublish_track = 6; UnpublishTrackRequest unpublish_track = 6;
PublishDataRequest publish_data = 7; PublishDataRequest publish_data = 7;
SetSubscribedRequest set_subscribed = 8; SetSubscribedRequest set_subscribed = 8;
UpdateLocalMetadataRequest update_local_metadata = 9; SetLocalMetadataRequest set_local_metadata = 9;
UpdateLocalNameRequest update_local_name = 10; SetLocalNameRequest set_local_name = 10;
GetSessionStatsRequest get_session_stats = 11; SetLocalAttributesRequest set_local_attributes = 11;
GetSessionStatsRequest get_session_stats = 12;
PublishTranscriptionRequest publish_transcription = 13;
PublishSipDtmfRequest publish_sip_dtmf = 14;
// Track // Track
CreateVideoTrackRequest create_video_track = 12; CreateVideoTrackRequest create_video_track = 15;
CreateAudioTrackRequest create_audio_track = 13; CreateAudioTrackRequest create_audio_track = 16;
GetStatsRequest get_stats = 14; LocalTrackMuteRequest local_track_mute = 17;
EnableRemoteTrackRequest enable_remote_track = 18;
GetStatsRequest get_stats = 19;
// Video // Video
NewVideoStreamRequest new_video_stream = 16; NewVideoStreamRequest new_video_stream = 20;
NewVideoSourceRequest new_video_source = 17; NewVideoSourceRequest new_video_source = 21;
CaptureVideoFrameRequest capture_video_frame = 18; CaptureVideoFrameRequest capture_video_frame = 22;
VideoConvertRequest video_convert = 19; VideoConvertRequest video_convert = 23;
VideoStreamFromParticipantRequest video_stream_from_participant = 24;
// Audio // Audio
NewAudioStreamRequest new_audio_stream = 22; NewAudioStreamRequest new_audio_stream = 25;
NewAudioSourceRequest new_audio_source = 23; NewAudioSourceRequest new_audio_source = 26;
CaptureAudioFrameRequest capture_audio_frame = 24; CaptureAudioFrameRequest capture_audio_frame = 27;
NewAudioResamplerRequest new_audio_resampler = 25; ClearAudioBufferRequest clear_audio_buffer = 28;
RemixAndResampleRequest remix_and_resample = 26; NewAudioResamplerRequest new_audio_resampler = 29;
E2eeRequest e2ee = 27; RemixAndResampleRequest remix_and_resample = 30;
E2eeRequest e2ee = 31;
AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
NewSoxResamplerRequest new_sox_resampler = 33;
PushSoxResamplerRequest push_sox_resampler = 34;
FlushSoxResamplerRequest flush_sox_resampler = 35;
SendChatMessageRequest send_chat_message = 36;
EditChatMessageRequest edit_chat_message = 37;
// RPC
PerformRpcRequest perform_rpc = 38;
RegisterRpcMethodRequest register_rpc_method = 39;
UnregisterRpcMethodRequest unregister_rpc_method = 40;
RpcMethodInvocationResponseRequest rpc_method_invocation_response = 41;
} }
} }
@@ -100,28 +121,45 @@ message FfiResponse {
UnpublishTrackResponse unpublish_track = 6; UnpublishTrackResponse unpublish_track = 6;
PublishDataResponse publish_data = 7; PublishDataResponse publish_data = 7;
SetSubscribedResponse set_subscribed = 8; SetSubscribedResponse set_subscribed = 8;
UpdateLocalMetadataResponse update_local_metadata = 9; SetLocalMetadataResponse set_local_metadata = 9;
UpdateLocalNameResponse update_local_name = 10; SetLocalNameResponse set_local_name = 10;
GetSessionStatsResponse get_session_stats = 11; SetLocalAttributesResponse set_local_attributes = 11;
GetSessionStatsResponse get_session_stats = 12;
PublishTranscriptionResponse publish_transcription = 13;
PublishSipDtmfResponse publish_sip_dtmf = 14;
// Track // Track
CreateVideoTrackResponse create_video_track = 12; CreateVideoTrackResponse create_video_track = 15;
CreateAudioTrackResponse create_audio_track = 13; CreateAudioTrackResponse create_audio_track = 16;
GetStatsResponse get_stats = 14; LocalTrackMuteResponse local_track_mute = 17;
EnableRemoteTrackResponse enable_remote_track = 18;
GetStatsResponse get_stats = 19;
// Video // Video
NewVideoStreamResponse new_video_stream = 16; NewVideoStreamResponse new_video_stream = 20;
NewVideoSourceResponse new_video_source = 17; NewVideoSourceResponse new_video_source = 21;
CaptureVideoFrameResponse capture_video_frame = 18; CaptureVideoFrameResponse capture_video_frame = 22;
VideoConvertResponse video_convert = 19; VideoConvertResponse video_convert = 23;
VideoStreamFromParticipantResponse video_stream_from_participant = 24;
// Audio // Audio
NewAudioStreamResponse new_audio_stream = 22; NewAudioStreamResponse new_audio_stream = 25;
NewAudioSourceResponse new_audio_source = 23; NewAudioSourceResponse new_audio_source = 26;
CaptureAudioFrameResponse capture_audio_frame = 24; CaptureAudioFrameResponse capture_audio_frame = 27;
NewAudioResamplerResponse new_audio_resampler = 25; ClearAudioBufferResponse clear_audio_buffer = 28;
RemixAndResampleResponse remix_and_resample = 26; NewAudioResamplerResponse new_audio_resampler = 29;
E2eeResponse e2ee = 27; RemixAndResampleResponse remix_and_resample = 30;
AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
E2eeResponse e2ee = 32;
NewSoxResamplerResponse new_sox_resampler = 33;
PushSoxResamplerResponse push_sox_resampler = 34;
FlushSoxResamplerResponse flush_sox_resampler = 35;
SendChatMessageResponse send_chat_message = 36;
// RPC
PerformRpcResponse perform_rpc = 37;
RegisterRpcMethodResponse register_rpc_method = 38;
UnregisterRpcMethodResponse unregister_rpc_method = 39;
RpcMethodInvocationResponseResponse rpc_method_invocation_response = 40;
} }
} }
@@ -135,18 +173,24 @@ message FfiEvent {
VideoStreamEvent video_stream_event = 3; VideoStreamEvent video_stream_event = 3;
AudioStreamEvent audio_stream_event = 4; AudioStreamEvent audio_stream_event = 4;
ConnectCallback connect = 5; ConnectCallback connect = 5;
DisconnectCallback disconnect = 6; DisconnectCallback disconnect = 7;
DisposeCallback dispose = 7; DisposeCallback dispose = 8;
PublishTrackCallback publish_track = 8; PublishTrackCallback publish_track = 9;
UnpublishTrackCallback unpublish_track = 9; UnpublishTrackCallback unpublish_track = 10;
PublishDataCallback publish_data = 10; PublishDataCallback publish_data = 11;
CaptureAudioFrameCallback capture_audio_frame = 11; PublishTranscriptionCallback publish_transcription = 12;
UpdateLocalMetadataCallback update_local_metadata = 12; CaptureAudioFrameCallback capture_audio_frame = 13;
UpdateLocalNameCallback update_local_name = 13; SetLocalMetadataCallback set_local_metadata = 14;
GetStatsCallback get_stats = 14; SetLocalNameCallback set_local_name = 15;
LogBatch logs = 15; SetLocalAttributesCallback set_local_attributes = 16;
GetSessionStatsCallback get_session_stats = 16; GetStatsCallback get_stats = 17;
Panic panic = 17; LogBatch logs = 18;
GetSessionStatsCallback get_session_stats = 19;
Panic panic = 20;
PublishSipDtmfCallback publish_sip_dtmf = 21;
SendChatMessageCallback chat_message = 22;
PerformRpcCallback perform_rpc = 23;
RpcMethodInvocationEvent rpc_method_invocation = 24;
} }
} }
@@ -154,14 +198,14 @@ message FfiEvent {
// e.g: This is used for the Unity Editor after each assemblies reload. // e.g: This is used for the Unity Editor after each assemblies reload.
// TODO(theomonnom): Implement a debug mode where we can find all leaked handles? // TODO(theomonnom): Implement a debug mode where we can find all leaked handles?
message DisposeRequest { message DisposeRequest {
bool async = 1; required bool async = 1;
} }
message DisposeResponse { message DisposeResponse {
optional uint64 async_id = 1; // None if sync optional uint64 async_id = 1; // None if sync
} }
message DisposeCallback { message DisposeCallback {
uint64 async_id = 1; required uint64 async_id = 1;
} }
enum LogLevel { enum LogLevel {
@@ -173,12 +217,12 @@ enum LogLevel {
} }
message LogRecord { message LogRecord {
LogLevel level = 1; required LogLevel level = 1;
string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc... required string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc...
optional string module_path = 3; optional string module_path = 3;
optional string file = 4; optional string file = 4;
optional uint32 line = 5; optional uint32 line = 5;
string message = 6; required string message = 6;
} }
message LogBatch { message LogBatch {
@@ -186,7 +230,7 @@ message LogBatch {
} }
message Panic { message Panic {
string message = 1; required string message = 1;
} }
// TODO(theomonnom): Debug messages (Print handles). // TODO(theomonnom): Debug messages (Print handles).

4
src/protocols/handle.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -27,5 +27,5 @@ option csharp_namespace = "LiveKit.Proto";
// When refering to a handle without owning it, we just use a uint32 without this message. // When refering to a handle without owning it, we just use a uint32 without this message.
// (the variable name is suffixed with "_handle") // (the variable name is suffixed with "_handle")
message FfiOwnedHandle { message FfiOwnedHandle {
uint64 id = 1; required uint64 id = 1;
} }

24
src/protocols/participant.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -20,13 +20,23 @@ option csharp_namespace = "LiveKit.Proto";
import "handle.proto"; import "handle.proto";
message ParticipantInfo { message ParticipantInfo {
string sid = 1; required string sid = 1;
string name = 2; required string name = 2;
string identity = 3; required string identity = 3;
string metadata = 4; required string metadata = 4;
map<string, string> attributes = 5;
required ParticipantKind kind = 6;
} }
message OwnedParticipant { message OwnedParticipant {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
ParticipantInfo info = 2; required ParticipantInfo info = 2;
}
enum ParticipantKind {
PARTICIPANT_KIND_STANDARD = 0;
PARTICIPANT_KIND_INGRESS = 1;
PARTICIPANT_KIND_EGRESS = 2;
PARTICIPANT_KIND_SIP = 3;
PARTICIPANT_KIND_AGENT = 4;
} }

443
src/protocols/room.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -26,147 +26,240 @@ import "stats.proto";
// Connect to a new LiveKit room // Connect to a new LiveKit room
message ConnectRequest { message ConnectRequest {
string url = 1; required string url = 1;
string token = 2; required string token = 2;
RoomOptions options = 3; required RoomOptions options = 3;
} }
message ConnectResponse { message ConnectResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message ConnectCallback { message ConnectCallback {
message ParticipantWithTracks { message ParticipantWithTracks {
OwnedParticipant participant = 1; required OwnedParticipant participant = 1;
// TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send // TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send
// a TrackSubscribed event // a TrackSubscribed event
repeated OwnedTrackPublication publications = 2; repeated OwnedTrackPublication publications = 2;
} }
uint64 async_id = 1; message Result {
optional string error = 2; required OwnedRoom room = 1;
OwnedRoom room = 3; required OwnedParticipant local_participant = 2;
OwnedParticipant local_participant = 4; repeated ParticipantWithTracks participants = 3;
repeated ParticipantWithTracks participants = 5; }
required uint64 async_id = 1;
oneof message {
string error = 2;
Result result = 3;
}
} }
// Disconnect from the a room // Disconnect from the a room
message DisconnectRequest { uint64 room_handle = 1; } message DisconnectRequest { required uint64 room_handle = 1; }
message DisconnectResponse { uint64 async_id = 1; } message DisconnectResponse { required uint64 async_id = 1; }
message DisconnectCallback { uint64 async_id = 1; } message DisconnectCallback { required uint64 async_id = 1; }
// Publish a track to the room // Publish a track to the room
message PublishTrackRequest { message PublishTrackRequest {
uint64 local_participant_handle = 1; required uint64 local_participant_handle = 1;
uint64 track_handle = 2; required uint64 track_handle = 2;
TrackPublishOptions options = 3; required TrackPublishOptions options = 3;
} }
message PublishTrackResponse { message PublishTrackResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message PublishTrackCallback { message PublishTrackCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2; oneof message {
OwnedTrackPublication publication = 3; string error = 2;
OwnedTrackPublication publication = 3;
}
} }
// Unpublish a track from the room // Unpublish a track from the room
message UnpublishTrackRequest { message UnpublishTrackRequest {
uint64 local_participant_handle = 1; required uint64 local_participant_handle = 1;
string track_sid = 2; required string track_sid = 2;
bool stop_on_unpublish = 3; required bool stop_on_unpublish = 3;
} }
message UnpublishTrackResponse { message UnpublishTrackResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message UnpublishTrackCallback { message UnpublishTrackCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2; optional string error = 2;
} }
// Publish data to other participants // Publish data to other participants
message PublishDataRequest { message PublishDataRequest {
uint64 local_participant_handle = 1; required uint64 local_participant_handle = 1;
uint64 data_ptr = 2; required uint64 data_ptr = 2;
uint64 data_len = 3; required uint64 data_len = 3;
DataPacketKind kind = 4; required bool reliable = 4;
repeated string destination_sids = 5; // destination repeated string destination_sids = 5 [deprecated=true];
optional string topic = 6; optional string topic = 6;
repeated string destination_identities = 7;
} }
message PublishDataResponse { message PublishDataResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message PublishDataCallback { message PublishDataCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2;
}
// Publish transcription messages to room
message PublishTranscriptionRequest {
required uint64 local_participant_handle = 1;
required string participant_identity = 2;
required string track_id = 3;
repeated TranscriptionSegment segments = 4;
}
message PublishTranscriptionResponse {
required uint64 async_id = 1;
}
message PublishTranscriptionCallback {
required uint64 async_id = 1;
optional string error = 2;
}
// Publish Sip DTMF messages to other participants
message PublishSipDtmfRequest {
required uint64 local_participant_handle = 1;
required uint32 code = 2;
required string digit = 3;
repeated string destination_identities = 4;
}
message PublishSipDtmfResponse {
required uint64 async_id = 1;
}
message PublishSipDtmfCallback {
required uint64 async_id = 1;
optional string error = 2; optional string error = 2;
} }
// Change the local participant's metadata // Change the local participant's metadata
message UpdateLocalMetadataRequest { message SetLocalMetadataRequest {
uint64 local_participant_handle = 1; required uint64 local_participant_handle = 1;
string metadata = 2; required string metadata = 2;
} }
message UpdateLocalMetadataResponse { message SetLocalMetadataResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message UpdateLocalMetadataCallback { message SetLocalMetadataCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2;
}
message SendChatMessageRequest {
required uint64 local_participant_handle = 1;
required string message = 2;
repeated string destination_identities = 3;
optional string sender_identity = 4;
}
message EditChatMessageRequest {
required uint64 local_participant_handle = 1;
required string edit_text = 2;
required ChatMessage original_message = 3;
repeated string destination_identities = 4;
optional string sender_identity = 5;
}
message SendChatMessageResponse {
required uint64 async_id = 1;
}
message SendChatMessageCallback {
required uint64 async_id = 1;
oneof message {
string error = 2;
ChatMessage chat_message = 3;
}
}
// Change the local participant's attributes
message SetLocalAttributesRequest {
required uint64 local_participant_handle = 1;
repeated AttributesEntry attributes = 2;
}
message AttributesEntry {
required string key = 1;
required string value = 2;
}
message SetLocalAttributesResponse {
required uint64 async_id = 1;
}
message SetLocalAttributesCallback {
required uint64 async_id = 1;
optional string error = 2;
} }
// Change the local participant's name // Change the local participant's name
message UpdateLocalNameRequest { message SetLocalNameRequest {
uint64 local_participant_handle = 1; required uint64 local_participant_handle = 1;
string name = 2; required string name = 2;
} }
message UpdateLocalNameResponse { message SetLocalNameResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message UpdateLocalNameCallback { message SetLocalNameCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2;
} }
// Change the "desire" to subs2ribe to a track // Change the "desire" to subs2ribe to a track
message SetSubscribedRequest { message SetSubscribedRequest {
bool subscribe = 1; required bool subscribe = 1;
uint64 publication_handle = 2; required uint64 publication_handle = 2;
} }
message SetSubscribedResponse {} message SetSubscribedResponse {}
message GetSessionStatsRequest { message GetSessionStatsRequest {
uint64 room_handle = 1; required uint64 room_handle = 1;
} }
message GetSessionStatsResponse { message GetSessionStatsResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message GetSessionStatsCallback { message GetSessionStatsCallback {
uint64 async_id = 1; message Result {
optional string error = 2; repeated RtcStats publisher_stats = 1;
repeated RtcStats publisher_stats = 3; repeated RtcStats subscriber_stats = 2;
repeated RtcStats subscriber_stats = 4; }
}
required uint64 async_id = 1;
oneof message {
string error = 2;
Result result = 3;
}
}
// //
// Options // Options
// //
message VideoEncoding { message VideoEncoding {
uint64 max_bitrate = 1; required uint64 max_bitrate = 1;
double max_framerate = 2; required double max_framerate = 2;
} }
message AudioEncoding { message AudioEncoding {
uint64 max_bitrate = 1; required uint64 max_bitrate = 1;
} }
message TrackPublishOptions { message TrackPublishOptions {
// encodings are optional // encodings are optional
VideoEncoding video_encoding = 1; optional VideoEncoding video_encoding = 1;
AudioEncoding audio_encoding = 2; optional AudioEncoding audio_encoding = 2;
VideoCodec video_codec = 3; optional VideoCodec video_codec = 3;
bool dtx = 4; optional bool dtx = 4;
bool red = 5; optional bool red = 5;
bool simulcast = 6; optional bool simulcast = 6;
TrackSource source = 7; optional TrackSource source = 7;
optional string stream = 8;
} }
enum IceTransportType { enum IceTransportType {
@@ -182,8 +275,8 @@ enum ContinualGatheringPolicy {
message IceServer { message IceServer {
repeated string urls = 1; repeated string urls = 1;
string username = 2; optional string username = 2;
string password = 3; optional string password = 3;
} }
message RtcConfig { message RtcConfig {
@@ -193,12 +286,12 @@ message RtcConfig {
} }
message RoomOptions { message RoomOptions {
bool auto_subscribe = 1; optional bool auto_subscribe = 1;
bool adaptive_stream = 2; optional bool adaptive_stream = 2;
bool dynacast = 3; optional bool dynacast = 3;
optional E2eeOptions e2ee = 4; optional E2eeOptions e2ee = 4;
optional RtcConfig rtc_config = 5; // allow to setup a custom RtcConfiguration optional RtcConfig rtc_config = 5; // allow to setup a custom RtcConfiguration
uint32 join_retries = 6; optional uint32 join_retries = 6;
} }
// //
@@ -223,163 +316,237 @@ enum DataPacketKind {
KIND_RELIABLE = 1; KIND_RELIABLE = 1;
} }
enum DisconnectReason {
UNKNOWN_REASON = 0;
// the client initiated the disconnect
CLIENT_INITIATED = 1;
// another participant with the same identity has joined the room
DUPLICATE_IDENTITY = 2;
// the server instance is shutting down
SERVER_SHUTDOWN = 3;
// RoomService.RemoveParticipant was called
PARTICIPANT_REMOVED = 4;
// RoomService.DeleteRoom was called
ROOM_DELETED = 5;
// the client is attempting to resume a session, but server is not aware of it
STATE_MISMATCH = 6;
// client was unable to connect fully
JOIN_FAILURE = 7;
// Cloud-only, the server requested Participant to migrate the connection elsewhere
MIGRATION = 8;
// the signal websocket was closed unexpectedly
SIGNAL_CLOSE = 9;
// the room was closed, due to all Standard and Ingress participants having left
ROOM_CLOSED = 10;
}
message TranscriptionSegment {
required string id = 1;
required string text = 2;
required uint64 start_time = 3;
required uint64 end_time = 4;
required bool final = 5;
required string language = 6;
}
message BufferInfo { message BufferInfo {
uint64 data_ptr = 1; required uint64 data_ptr = 1;
uint64 data_len = 2; required uint64 data_len = 2;
} }
message OwnedBuffer { message OwnedBuffer {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
BufferInfo data = 2; required BufferInfo data = 2;
} }
message RoomEvent { message RoomEvent {
uint64 room_handle = 1; required uint64 room_handle = 1;
oneof message { oneof message {
ParticipantConnected participant_connected = 2; ParticipantConnected participant_connected = 2;
ParticipantDisconnected participant_disconnected = 3; ParticipantDisconnected participant_disconnected = 3;
LocalTrackPublished local_track_published = 4; LocalTrackPublished local_track_published = 4;
LocalTrackUnpublished local_track_unpublished = 5; LocalTrackUnpublished local_track_unpublished = 5;
TrackPublished track_published = 6; LocalTrackSubscribed local_track_subscribed = 6;
TrackUnpublished track_unpublished = 7; TrackPublished track_published = 7;
TrackSubscribed track_subscribed = 8; TrackUnpublished track_unpublished = 8;
TrackUnsubscribed track_unsubscribed = 9; TrackSubscribed track_subscribed = 9;
TrackSubscriptionFailed track_subscription_failed = 10; TrackUnsubscribed track_unsubscribed = 10;
TrackMuted track_muted = 11; TrackSubscriptionFailed track_subscription_failed = 11;
TrackUnmuted track_unmuted = 12; TrackMuted track_muted = 12;
ActiveSpeakersChanged active_speakers_changed = 13; TrackUnmuted track_unmuted = 13;
RoomMetadataChanged room_metadata_changed = 14; ActiveSpeakersChanged active_speakers_changed = 14;
ParticipantMetadataChanged participant_metadata_changed = 15; RoomMetadataChanged room_metadata_changed = 15;
ParticipantNameChanged participant_name_changed = 16; RoomSidChanged room_sid_changed = 16;
ConnectionQualityChanged connection_quality_changed = 17; ParticipantMetadataChanged participant_metadata_changed = 17;
ConnectionStateChanged connection_state_changed = 19; ParticipantNameChanged participant_name_changed = 18;
// Connected connected = 20; ParticipantAttributesChanged participant_attributes_changed = 19;
Disconnected disconnected = 21; ConnectionQualityChanged connection_quality_changed = 20;
Reconnecting reconnecting = 22; ConnectionStateChanged connection_state_changed = 21;
Reconnected reconnected = 23; // Connected connected = 21;
E2eeStateChanged e2ee_state_changed = 24; Disconnected disconnected = 22;
RoomEOS eos = 25; // The stream of room events has ended Reconnecting reconnecting = 23;
DataPacketReceived data_packet_received = 26; Reconnected reconnected = 24;
E2eeStateChanged e2ee_state_changed = 25;
RoomEOS eos = 26; // The stream of room events has ended
DataPacketReceived data_packet_received = 27;
TranscriptionReceived transcription_received = 28;
ChatMessageReceived chat_message = 29;
} }
} }
message RoomInfo { message RoomInfo {
string sid = 1; optional string sid = 1;
string name = 2; required string name = 2;
string metadata = 3; required string metadata = 3;
} }
message OwnedRoom { message OwnedRoom {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
RoomInfo info = 2; required RoomInfo info = 2;
} }
message ParticipantConnected { OwnedParticipant info = 1; } message ParticipantConnected { required OwnedParticipant info = 1; }
message ParticipantDisconnected { message ParticipantDisconnected {
string participant_sid = 1; required string participant_identity = 1;
} }
message LocalTrackPublished { message LocalTrackPublished {
// The TrackPublicationInfo comes from the PublishTrack response // The TrackPublicationInfo comes from the PublishTrack response
// and the FfiClient musts wait for it before firing this event // and the FfiClient musts wait for it before firing this event
string track_sid = 1; required string track_sid = 1;
} }
message LocalTrackUnpublished { message LocalTrackUnpublished {
string publication_sid = 1; required string publication_sid = 1;
}
message LocalTrackSubscribed {
required string track_sid = 2;
} }
message TrackPublished { message TrackPublished {
string participant_sid = 1; required string participant_identity = 1;
OwnedTrackPublication publication = 2; required OwnedTrackPublication publication = 2;
} }
message TrackUnpublished { message TrackUnpublished {
string participant_sid = 1; required string participant_identity = 1;
string publication_sid = 2; required string publication_sid = 2;
} }
// Publication isn't needed for subscription events on the FFI // Publication isn't needed for subscription events on the FFI
// The FFI will retrieve the publication using the Track sid // The FFI will retrieve the publication using the Track sid
message TrackSubscribed { message TrackSubscribed {
string participant_sid = 1; required string participant_identity = 1;
OwnedTrack track = 2; required OwnedTrack track = 2;
} }
message TrackUnsubscribed { message TrackUnsubscribed {
// The FFI language can dispose/remove the VideoSink here // The FFI language can dispose/remove the VideoSink here
string participant_sid = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
} }
message TrackSubscriptionFailed { message TrackSubscriptionFailed {
string participant_sid = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
string error = 3; required string error = 3;
} }
message TrackMuted { message TrackMuted {
string participant_sid = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
} }
message TrackUnmuted { message TrackUnmuted {
string participant_sid = 1; required string participant_identity = 1;
string track_sid = 2; required string track_sid = 2;
} }
message E2eeStateChanged { message E2eeStateChanged {
string participant_sid = 1; // Using sid instead of identity for ffi communication required string participant_identity = 1; // Using sid instead of identity for ffi communication
EncryptionState state = 2; required EncryptionState state = 2;
} }
message ActiveSpeakersChanged { repeated string participant_sids = 1; } message ActiveSpeakersChanged { repeated string participant_identities = 1; }
message RoomMetadataChanged { message RoomMetadataChanged {
string metadata = 1; required string metadata = 1;
}
message RoomSidChanged {
required string sid = 1;
} }
message ParticipantMetadataChanged { message ParticipantMetadataChanged {
string participant_sid = 1; required string participant_identity = 1;
string metadata = 2; required string metadata = 2;
}
message ParticipantAttributesChanged {
required string participant_identity = 1;
repeated AttributesEntry attributes = 2;
repeated AttributesEntry changed_attributes = 3;
} }
message ParticipantNameChanged { message ParticipantNameChanged {
string participant_sid = 1; required string participant_identity = 1;
string name = 2; required string name = 2;
} }
message ConnectionQualityChanged { message ConnectionQualityChanged {
string participant_sid = 1; required string participant_identity = 1;
ConnectionQuality quality = 2; required ConnectionQuality quality = 2;
} }
message UserPacket { message UserPacket {
OwnedBuffer data = 1; required OwnedBuffer data = 1;
optional string topic = 2; optional string topic = 2;
} }
message ChatMessage {
required string id = 1;
required int64 timestamp = 2;
required string message = 3;
optional int64 edit_timestamp = 4;
optional bool deleted = 5;
optional bool generated = 6;
}
message ChatMessageReceived {
required ChatMessage message = 1;
required string participant_identity = 2;
}
message SipDTMF { message SipDTMF {
uint32 code = 1; required uint32 code = 1;
optional string digit = 2; optional string digit = 2;
} }
message DataPacketReceived { message DataPacketReceived {
DataPacketKind kind = 1; required DataPacketKind kind = 1;
string participant_identity = 2; // Can be empty if the data is sent a server SDK required string participant_identity = 2; // Can be empty if the data is sent a server SDK
optional string participant_sid = 3 [deprecated=true]; // Can be empty if the data is sent a server SDK
oneof value { oneof value {
UserPacket user = 4; UserPacket user = 4;
SipDTMF sip_dtmf = 5; SipDTMF sip_dtmf = 5;
} }
} }
message ConnectionStateChanged { ConnectionState state = 1; } message TranscriptionReceived {
optional string participant_identity = 1;
optional string track_sid = 2;
repeated TranscriptionSegment segments = 3;
}
message ConnectionStateChanged { required ConnectionState state = 1; }
message Connected {} message Connected {}
message Disconnected {} message Disconnected {
required DisconnectReason reason = 1;
}
message Reconnecting {} message Reconnecting {}
message Reconnected {} message Reconnected {}
message RoomEOS {} message RoomEOS {}

81
src/protocols/rpc.proto Normal file
View File

@@ -0,0 +1,81 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
message RpcError {
required uint32 code = 1;
required string message = 2;
optional string data = 3;
}
// FFI Requests
message PerformRpcRequest {
required uint64 local_participant_handle = 1;
required string destination_identity = 2;
required string method = 3;
required string payload = 4;
optional uint32 response_timeout_ms = 5;
}
message RegisterRpcMethodRequest {
required uint64 local_participant_handle = 1;
required string method = 2;
}
message UnregisterRpcMethodRequest {
required uint64 local_participant_handle = 1;
required string method = 2;
}
message RpcMethodInvocationResponseRequest {
required uint64 local_participant_handle = 1;
required uint64 invocation_id = 2;
optional string payload = 3;
optional RpcError error = 4;
}
// FFI Responses
message PerformRpcResponse {
required uint64 async_id = 1;
}
message RegisterRpcMethodResponse {}
message UnregisterRpcMethodResponse {}
message RpcMethodInvocationResponseResponse {
optional string error = 1;
}
// FFI Callbacks
message PerformRpcCallback {
required uint64 async_id = 1;
optional string payload = 2;
optional RpcError error = 3;
}
// FFI Events
message RpcMethodInvocationEvent {
required uint64 local_participant_handle = 1;
required uint64 invocation_id = 2;
required string method = 3;
required string request_id = 4;
required string caller_identity = 5;
required string payload = 6;
required uint32 response_timeout_ms = 7;
}

456
src/protocols/stats.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -91,83 +91,83 @@ enum IceTcpCandidateType {
message RtcStats { message RtcStats {
message Codec { message Codec {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
CodecStats codec = 2; required CodecStats codec = 2;
} }
message InboundRtp { message InboundRtp {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
RtpStreamStats stream = 2; required RtpStreamStats stream = 2;
ReceivedRtpStreamStats received = 3; required ReceivedRtpStreamStats received = 3;
InboundRtpStreamStats inbound = 4; required InboundRtpStreamStats inbound = 4;
} }
message OutboundRtp { message OutboundRtp {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
RtpStreamStats stream = 2; required RtpStreamStats stream = 2;
SentRtpStreamStats sent = 3; required SentRtpStreamStats sent = 3;
OutboundRtpStreamStats outbound = 4; required OutboundRtpStreamStats outbound = 4;
} }
message RemoteInboundRtp { message RemoteInboundRtp {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
RtpStreamStats stream = 2; required RtpStreamStats stream = 2;
ReceivedRtpStreamStats received = 3; required ReceivedRtpStreamStats received = 3;
RemoteInboundRtpStreamStats remote_inbound = 4; required RemoteInboundRtpStreamStats remote_inbound = 4;
} }
message RemoteOutboundRtp { message RemoteOutboundRtp {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
RtpStreamStats stream = 2; required RtpStreamStats stream = 2;
SentRtpStreamStats sent = 3; required SentRtpStreamStats sent = 3;
RemoteOutboundRtpStreamStats remote_outbound = 4; required RemoteOutboundRtpStreamStats remote_outbound = 4;
} }
message MediaSource { message MediaSource {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
MediaSourceStats source = 2; required MediaSourceStats source = 2;
AudioSourceStats audio = 3; required AudioSourceStats audio = 3;
VideoSourceStats video = 4; required VideoSourceStats video = 4;
} }
message MediaPlayout { message MediaPlayout {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
AudioPlayoutStats audio_playout = 2; required AudioPlayoutStats audio_playout = 2;
} }
message PeerConnection { message PeerConnection {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
PeerConnectionStats pc = 2; required PeerConnectionStats pc = 2;
} }
message DataChannel { message DataChannel {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
DataChannelStats dc = 2; required DataChannelStats dc = 2;
} }
message Transport { message Transport {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
TransportStats transport = 2; required TransportStats transport = 2;
} }
message CandidatePair { message CandidatePair {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
CandidatePairStats candidate_pair = 2; required CandidatePairStats candidate_pair = 2;
} }
message LocalCandidate { message LocalCandidate {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
IceCandidateStats candidate = 2; required IceCandidateStats candidate = 2;
} }
message RemoteCandidate { message RemoteCandidate {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
IceCandidateStats candidate = 2; required IceCandidateStats candidate = 2;
} }
message Certificate { message Certificate {
RtcStatsData rtc = 1; required RtcStatsData rtc = 1;
CertificateStats certificate = 2; required CertificateStats certificate = 2;
} }
message Track { message Track {
@@ -194,256 +194,256 @@ message RtcStats {
} }
message RtcStatsData { message RtcStatsData {
string id = 1; required string id = 1;
int64 timestamp = 2; required int64 timestamp = 2;
} }
message CodecStats { message CodecStats {
uint32 payload_type = 1; required uint32 payload_type = 1;
string transport_id = 2; required string transport_id = 2;
string mime_type = 3; required string mime_type = 3;
uint32 clock_rate = 4; required uint32 clock_rate = 4;
uint32 channels = 5; required uint32 channels = 5;
string sdp_fmtp_line = 6; required string sdp_fmtp_line = 6;
} }
message RtpStreamStats { message RtpStreamStats {
uint32 ssrc = 1; required uint32 ssrc = 1;
string kind = 2; required string kind = 2;
string transport_id = 3; required string transport_id = 3;
string codec_id = 4; required string codec_id = 4;
} }
message ReceivedRtpStreamStats { message ReceivedRtpStreamStats {
uint64 packets_received = 1; required uint64 packets_received = 1;
int64 packets_lost = 2; required int64 packets_lost = 2;
double jitter = 3; required double jitter = 3;
} }
message InboundRtpStreamStats { message InboundRtpStreamStats {
string track_identifier = 1; required string track_identifier = 1;
string mid = 2; required string mid = 2;
string remote_id = 3; required string remote_id = 3;
uint32 frames_decoded = 4; required uint32 frames_decoded = 4;
uint32 key_frames_decoded = 5; required uint32 key_frames_decoded = 5;
uint32 frames_rendered = 6; required uint32 frames_rendered = 6;
uint32 frames_dropped = 7; required uint32 frames_dropped = 7;
uint32 frame_width = 8; required uint32 frame_width = 8;
uint32 frame_height = 9; required uint32 frame_height = 9;
double frames_per_second = 10; required double frames_per_second = 10;
uint64 qp_sum = 11; required uint64 qp_sum = 11;
double total_decode_time = 12; required double total_decode_time = 12;
double total_inter_frame_delay = 13; required double total_inter_frame_delay = 13;
double total_squared_inter_frame_delay = 14; required double total_squared_inter_frame_delay = 14;
uint32 pause_count = 15; required uint32 pause_count = 15;
double total_pause_duration = 16; required double total_pause_duration = 16;
uint32 freeze_count = 17; required uint32 freeze_count = 17;
double total_freeze_duration = 18; required double total_freeze_duration = 18;
double last_packet_received_timestamp = 19; required double last_packet_received_timestamp = 19;
uint64 header_bytes_received = 20; required uint64 header_bytes_received = 20;
uint64 packets_discarded = 21; required uint64 packets_discarded = 21;
uint64 fec_bytes_received = 22; required uint64 fec_bytes_received = 22;
uint64 fec_packets_received = 23; required uint64 fec_packets_received = 23;
uint64 fec_packets_discarded = 24; required uint64 fec_packets_discarded = 24;
uint64 bytes_received = 25; required uint64 bytes_received = 25;
uint32 nack_count = 26; required uint32 nack_count = 26;
uint32 fir_count = 27; required uint32 fir_count = 27;
uint32 pli_count = 28; required uint32 pli_count = 28;
double total_processing_delay = 29; required double total_processing_delay = 29;
double estimated_playout_timestamp = 30; required double estimated_playout_timestamp = 30;
double jitter_buffer_delay = 31; required double jitter_buffer_delay = 31;
double jitter_buffer_target_delay = 32; required double jitter_buffer_target_delay = 32;
uint64 jitter_buffer_emitted_count = 33; required uint64 jitter_buffer_emitted_count = 33;
double jitter_buffer_minimum_delay = 34; required double jitter_buffer_minimum_delay = 34;
uint64 total_samples_received = 35; required uint64 total_samples_received = 35;
uint64 concealed_samples = 36; required uint64 concealed_samples = 36;
uint64 silent_concealed_samples = 37; required uint64 silent_concealed_samples = 37;
uint64 concealment_events = 38; required uint64 concealment_events = 38;
uint64 inserted_samples_for_deceleration = 39; required uint64 inserted_samples_for_deceleration = 39;
uint64 removed_samples_for_acceleration = 40; required uint64 removed_samples_for_acceleration = 40;
double audio_level = 41; required double audio_level = 41;
double total_audio_energy = 42; required double total_audio_energy = 42;
double total_samples_duration = 43; required double total_samples_duration = 43;
uint64 frames_received = 44; required uint64 frames_received = 44;
string decoder_implementation = 45; required string decoder_implementation = 45;
string playout_id = 46; required string playout_id = 46;
bool power_efficient_decoder = 47; required bool power_efficient_decoder = 47;
uint64 frames_assembled_from_multiple_packets = 48; required uint64 frames_assembled_from_multiple_packets = 48;
double total_assembly_time = 49; required double total_assembly_time = 49;
uint64 retransmitted_packets_received = 50; required uint64 retransmitted_packets_received = 50;
uint64 retransmitted_bytes_received = 51; required uint64 retransmitted_bytes_received = 51;
uint32 rtx_ssrc = 52; required uint32 rtx_ssrc = 52;
uint32 fec_ssrc = 53; required uint32 fec_ssrc = 53;
} }
message SentRtpStreamStats { message SentRtpStreamStats {
uint64 packets_sent = 1; required uint64 packets_sent = 1;
uint64 bytes_sent = 2; required uint64 bytes_sent = 2;
} }
message OutboundRtpStreamStats { message OutboundRtpStreamStats {
string mid = 1; required string mid = 1;
string media_source_id = 2; required string media_source_id = 2;
string remote_id = 3; required string remote_id = 3;
string rid = 4; required string rid = 4;
uint64 header_bytes_sent = 5; required uint64 header_bytes_sent = 5;
uint64 retransmitted_packets_sent = 6; required uint64 retransmitted_packets_sent = 6;
uint64 retransmitted_bytes_sent = 7; required uint64 retransmitted_bytes_sent = 7;
uint32 rtx_ssrc = 8; required uint32 rtx_ssrc = 8;
double target_bitrate = 9; required double target_bitrate = 9;
uint64 total_encoded_bytes_target = 10; required uint64 total_encoded_bytes_target = 10;
uint32 frame_width = 11; required uint32 frame_width = 11;
uint32 frame_height = 12; required uint32 frame_height = 12;
double frames_per_second = 13; required double frames_per_second = 13;
uint32 frames_sent = 14; required uint32 frames_sent = 14;
uint32 huge_frames_sent = 15; required uint32 huge_frames_sent = 15;
uint32 frames_encoded = 16; required uint32 frames_encoded = 16;
uint32 key_frames_encoded = 17; required uint32 key_frames_encoded = 17;
uint64 qp_sum = 18; required uint64 qp_sum = 18;
double total_encode_time = 19; required double total_encode_time = 19;
double total_packet_send_delay = 20; required double total_packet_send_delay = 20;
QualityLimitationReason quality_limitation_reason = 21; required QualityLimitationReason quality_limitation_reason = 21;
map<string, double> quality_limitation_durations = 22; map<string, double> quality_limitation_durations = 22;
uint32 quality_limitation_resolution_changes = 23; required uint32 quality_limitation_resolution_changes = 23;
uint32 nack_count = 24; required uint32 nack_count = 24;
uint32 fir_count = 25; required uint32 fir_count = 25;
uint32 pli_count = 26; required uint32 pli_count = 26;
string encoder_implementation = 27; required string encoder_implementation = 27;
bool power_efficient_encoder = 28; required bool power_efficient_encoder = 28;
bool active = 29; required bool active = 29;
string scalibility_mode = 30; required string scalability_mode = 30;
} }
message RemoteInboundRtpStreamStats { message RemoteInboundRtpStreamStats {
string local_id = 1; required string local_id = 1;
double round_trip_time = 2; required double round_trip_time = 2;
double total_round_trip_time = 3; required double total_round_trip_time = 3;
double fraction_lost = 4; required double fraction_lost = 4;
uint64 round_trip_time_measurements = 5; required uint64 round_trip_time_measurements = 5;
} }
message RemoteOutboundRtpStreamStats { message RemoteOutboundRtpStreamStats {
string local_id = 1; required string local_id = 1;
double remote_timestamp = 2; required double remote_timestamp = 2;
uint64 reports_sent = 3; required uint64 reports_sent = 3;
double round_trip_time = 4; required double round_trip_time = 4;
double total_round_trip_time = 5; required double total_round_trip_time = 5;
uint64 round_trip_time_measurements = 6; required uint64 round_trip_time_measurements = 6;
} }
message MediaSourceStats { message MediaSourceStats {
string track_identifier = 1; required string track_identifier = 1;
string kind = 2; required string kind = 2;
} }
message AudioSourceStats { message AudioSourceStats {
double audio_level = 1; required double audio_level = 1;
double total_audio_energy = 2; required double total_audio_energy = 2;
double total_samples_duration = 3; required double total_samples_duration = 3;
double echo_return_loss = 4; required double echo_return_loss = 4;
double echo_return_loss_enhancement = 5; required double echo_return_loss_enhancement = 5;
double dropped_samples_duration = 6; required double dropped_samples_duration = 6;
uint32 dropped_samples_events = 7; required uint32 dropped_samples_events = 7;
double total_capture_delay = 8; required double total_capture_delay = 8;
uint64 total_samples_captured = 9; required uint64 total_samples_captured = 9;
} }
message VideoSourceStats { message VideoSourceStats {
uint32 width = 1; required uint32 width = 1;
uint32 height = 2; required uint32 height = 2;
uint32 frames = 3; required uint32 frames = 3;
double frames_per_second = 4; required double frames_per_second = 4;
} }
message AudioPlayoutStats { message AudioPlayoutStats {
string kind = 1; required string kind = 1;
double synthesized_samples_duration = 2; required double synthesized_samples_duration = 2;
uint32 synthesized_samples_events = 3; required uint32 synthesized_samples_events = 3;
double total_samples_duration = 4; required double total_samples_duration = 4;
double total_playout_delay = 5; required double total_playout_delay = 5;
uint64 total_samples_count = 6; required uint64 total_samples_count = 6;
} }
message PeerConnectionStats { message PeerConnectionStats {
uint32 data_channels_opened = 1; required uint32 data_channels_opened = 1;
uint32 data_channels_closed = 2; required uint32 data_channels_closed = 2;
} }
message DataChannelStats { message DataChannelStats {
string label = 1; required string label = 1;
string protocol = 2; required string protocol = 2;
int32 data_channel_identifier = 3; required int32 data_channel_identifier = 3;
optional DataChannelState state = 4; optional DataChannelState state = 4;
uint32 messages_sent = 5; required uint32 messages_sent = 5;
uint64 bytes_sent = 6; required uint64 bytes_sent = 6;
uint32 messages_received = 7; required uint32 messages_received = 7;
uint64 bytes_received = 8; required uint64 bytes_received = 8;
} }
message TransportStats { message TransportStats {
uint64 packets_sent = 1; required uint64 packets_sent = 1;
uint64 packets_received = 2; required uint64 packets_received = 2;
uint64 bytes_sent = 3; required uint64 bytes_sent = 3;
uint64 bytes_received = 4; required uint64 bytes_received = 4;
IceRole ice_role = 5; required IceRole ice_role = 5;
string ice_local_username_fragment = 6; required string ice_local_username_fragment = 6;
optional DtlsTransportState dtls_state = 7; optional DtlsTransportState dtls_state = 7;
optional IceTransportState ice_state = 8; optional IceTransportState ice_state = 8;
string selected_candidate_pair_id = 9; required string selected_candidate_pair_id = 9;
string local_certificate_id = 10; required string local_certificate_id = 10;
string remote_certificate_id = 11; required string remote_certificate_id = 11;
string tls_version = 12; required string tls_version = 12;
string dtls_cipher = 13; required string dtls_cipher = 13;
DtlsRole dtls_role = 14; required DtlsRole dtls_role = 14;
string srtp_cipher = 15; required string srtp_cipher = 15;
uint32 selected_candidate_pair_changes = 16; required uint32 selected_candidate_pair_changes = 16;
} }
message CandidatePairStats { message CandidatePairStats {
string transport_id = 1; required string transport_id = 1;
string local_candidate_id = 2; required string local_candidate_id = 2;
string remote_candidate_id = 3; required string remote_candidate_id = 3;
optional IceCandidatePairState state = 4; optional IceCandidatePairState state = 4;
bool nominated = 5; required bool nominated = 5;
uint64 packets_sent = 6; required uint64 packets_sent = 6;
uint64 packets_received = 7; required uint64 packets_received = 7;
uint64 bytes_sent = 8; required uint64 bytes_sent = 8;
uint64 bytes_received = 9; required uint64 bytes_received = 9;
double last_packet_sent_timestamp = 10; required double last_packet_sent_timestamp = 10;
double last_packet_received_timestamp = 11; required double last_packet_received_timestamp = 11;
double total_round_trip_time = 12; required double total_round_trip_time = 12;
double current_round_trip_time = 13; required double current_round_trip_time = 13;
double available_outgoing_bitrate = 14; required double available_outgoing_bitrate = 14;
double available_incoming_bitrate = 15; required double available_incoming_bitrate = 15;
uint64 requests_received = 16; required uint64 requests_received = 16;
uint64 requests_sent = 17; required uint64 requests_sent = 17;
uint64 responses_received = 18; required uint64 responses_received = 18;
uint64 responses_sent = 19; required uint64 responses_sent = 19;
uint64 consent_requests_sent = 20; required uint64 consent_requests_sent = 20;
uint32 packets_discarded_on_send = 21; required uint32 packets_discarded_on_send = 21;
uint64 bytes_discarded_on_send = 22; required uint64 bytes_discarded_on_send = 22;
} }
message IceCandidateStats { message IceCandidateStats {
string transport_id = 1; required string transport_id = 1;
string address = 2; required string address = 2;
int32 port = 3; required int32 port = 3;
string protocol = 4; required string protocol = 4;
optional IceCandidateType candidate_type = 5; optional IceCandidateType candidate_type = 5;
int32 priority = 6; required int32 priority = 6;
string url = 7; required string url = 7;
optional IceServerTransportProtocol relay_protocol = 8; optional IceServerTransportProtocol relay_protocol = 8;
string foundation = 9; required string foundation = 9;
string related_address = 10; required string related_address = 10;
int32 related_port = 11; required int32 related_port = 11;
string username_fragment = 12; required string username_fragment = 12;
optional IceTcpCandidateType tcp_type = 13; optional IceTcpCandidateType tcp_type = 13;
} }
message CertificateStats { message CertificateStats {
string fingerprint = 1; required string fingerprint = 1;
string fingerprint_algorithm = 2; required string fingerprint_algorithm = 2;
string base64_certificate = 3; required string base64_certificate = 3;
string issuer_certificate_id = 4; required string issuer_certificate_id = 4;
} }

82
src/protocols/track.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
@@ -23,30 +23,30 @@ import "stats.proto";
// Create a new VideoTrack from a VideoSource // Create a new VideoTrack from a VideoSource
message CreateVideoTrackRequest { message CreateVideoTrackRequest {
string name = 1; required string name = 1;
uint64 source_handle = 2; required uint64 source_handle = 2;
} }
message CreateVideoTrackResponse { message CreateVideoTrackResponse {
OwnedTrack track = 1; required OwnedTrack track = 1;
} }
// Create a new AudioTrack from a AudioSource // Create a new AudioTrack from a AudioSource
message CreateAudioTrackRequest { message CreateAudioTrackRequest {
string name = 1; required string name = 1;
uint64 source_handle = 2; required uint64 source_handle = 2;
} }
message CreateAudioTrackResponse { message CreateAudioTrackResponse {
OwnedTrack track = 1; required OwnedTrack track = 1;
} }
message GetStatsRequest { message GetStatsRequest {
uint64 track_handle = 1; required uint64 track_handle = 1;
} }
message GetStatsResponse { message GetStatsResponse {
uint64 async_id = 1; required uint64 async_id = 1;
} }
message GetStatsCallback { message GetStatsCallback {
uint64 async_id = 1; required uint64 async_id = 1;
optional string error = 2; optional string error = 2;
repeated RtcStats stats = 3; repeated RtcStats stats = 3;
} }
@@ -78,34 +78,54 @@ enum StreamState {
} }
message TrackPublicationInfo { message TrackPublicationInfo {
string sid = 1; required string sid = 1;
string name = 2; required string name = 2;
TrackKind kind = 3; required TrackKind kind = 3;
TrackSource source = 4; required TrackSource source = 4;
bool simulcasted = 5; required bool simulcasted = 5;
uint32 width = 6; required uint32 width = 6;
uint32 height = 7; required uint32 height = 7;
string mime_type = 8; required string mime_type = 8;
bool muted = 9; required bool muted = 9;
bool remote = 10; required bool remote = 10;
EncryptionType encryption_type = 11; required EncryptionType encryption_type = 11;
} }
message OwnedTrackPublication { message OwnedTrackPublication {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
TrackPublicationInfo info = 2; required TrackPublicationInfo info = 2;
} }
message TrackInfo { message TrackInfo {
string sid = 1; required string sid = 1;
string name = 2; required string name = 2;
TrackKind kind = 3; required TrackKind kind = 3;
StreamState stream_state = 4; required StreamState stream_state = 4;
bool muted = 5; required bool muted = 5;
bool remote = 6; required bool remote = 6;
} }
message OwnedTrack { message OwnedTrack {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
TrackInfo info = 2; required TrackInfo info = 2;
}
// Mute/UnMute a track
message LocalTrackMuteRequest {
required uint64 track_handle = 1;
required bool mute = 2;
}
message LocalTrackMuteResponse {
required bool muted = 1;
}
// Enable/Disable a remote track
message EnableRemoteTrackRequest {
required uint64 track_handle = 1;
required bool enabled = 2;
}
message EnableRemoteTrackResponse {
required bool enabled = 1;
} }

100
src/protocols/video_frame.proto
View File

@@ -12,52 +12,66 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
syntax = "proto3"; syntax = "proto2";
package livekit.proto; package livekit.proto;
option csharp_namespace = "LiveKit.Proto"; option csharp_namespace = "LiveKit.Proto";
import "handle.proto"; import "handle.proto";
import "track.proto";
// Create a new VideoStream // Create a new VideoStream
// VideoStream is used to receive video frames from a track // VideoStream is used to receive video frames from a track
message NewVideoStreamRequest { message NewVideoStreamRequest {
uint64 track_handle = 1; required uint64 track_handle = 1;
VideoStreamType type = 2; required VideoStreamType type = 2;
// Get the frame on a specific format // Get the frame on a specific format
optional VideoBufferType format = 3; optional VideoBufferType format = 3;
bool normalize_stride = 4; // if true, stride will be set to width/chroma_width optional bool normalize_stride = 4; // if true, stride will be set to width/chroma_width
} }
message NewVideoStreamResponse { OwnedVideoStream stream = 1; } message NewVideoStreamResponse { required OwnedVideoStream stream = 1; }
// Request a video stream from a participant
message VideoStreamFromParticipantRequest {
required uint64 participant_handle = 1;
required VideoStreamType type = 2;
required TrackSource track_source = 3;
optional VideoBufferType format = 4;
optional bool normalize_stride = 5;
}
message VideoStreamFromParticipantResponse { required OwnedVideoStream stream = 1;}
// Create a new VideoSource // Create a new VideoSource
// VideoSource is used to send video frame to a track // VideoSource is used to send video frame to a track
message NewVideoSourceRequest { message NewVideoSourceRequest {
VideoSourceType type = 1; required VideoSourceType type = 1;
// Used to determine which encodings to use + simulcast layers // Used to determine which encodings to use + simulcast layers
// Most of the time it corresponds to the source resolution // Most of the time it corresponds to the source resolution
VideoSourceResolution resolution = 2; required VideoSourceResolution resolution = 2;
} }
message NewVideoSourceResponse { OwnedVideoSource source = 1; } message NewVideoSourceResponse { required OwnedVideoSource source = 1; }
// Push a frame to a VideoSource // Push a frame to a VideoSource
message CaptureVideoFrameRequest { message CaptureVideoFrameRequest {
uint64 source_handle = 1; required uint64 source_handle = 1;
VideoBufferInfo buffer = 2; required VideoBufferInfo buffer = 2;
int64 timestamp_us = 3; // In microseconds required int64 timestamp_us = 3; // In microseconds
VideoRotation rotation = 4; required VideoRotation rotation = 4;
} }
message CaptureVideoFrameResponse {} message CaptureVideoFrameResponse {}
message VideoConvertRequest { message VideoConvertRequest {
bool flip_y = 1; optional bool flip_y = 1;
VideoBufferInfo buffer = 2; required VideoBufferInfo buffer = 2;
VideoBufferType dst_type = 3; required VideoBufferType dst_type = 3;
} }
message VideoConvertResponse { message VideoConvertResponse {
optional string error = 1; oneof message {
OwnedVideoBuffer buffer = 2; string error = 1;
OwnedVideoBuffer buffer = 2;
}
} }
// //
@@ -65,9 +79,9 @@ message VideoConvertResponse {
// //
message VideoResolution { message VideoResolution {
uint32 width = 1; required uint32 width = 1;
uint32 height = 2; required uint32 height = 2;
double frame_rate = 3; required double frame_rate = 3;
} }
enum VideoCodec { enum VideoCodec {
@@ -100,21 +114,21 @@ enum VideoBufferType {
message VideoBufferInfo { message VideoBufferInfo {
message ComponentInfo { message ComponentInfo {
uint64 data_ptr = 1; required uint64 data_ptr = 1;
uint32 stride = 2; required uint32 stride = 2;
uint32 size = 3; required uint32 size = 3;
} }
VideoBufferType type = 1; required VideoBufferType type = 1;
uint32 width = 2; required uint32 width = 2;
uint32 height = 3; required uint32 height = 3;
uint64 data_ptr = 4; required uint64 data_ptr = 4;
uint32 stride = 6; // only for packed formats required uint32 stride = 6; // only for packed formats
repeated ComponentInfo components = 7; repeated ComponentInfo components = 7;
} }
message OwnedVideoBuffer { message OwnedVideoBuffer {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
VideoBufferInfo info = 2; required VideoBufferInfo info = 2;
} }
// //
@@ -128,16 +142,16 @@ enum VideoStreamType {
} }
message VideoStreamInfo { message VideoStreamInfo {
VideoStreamType type = 1; required VideoStreamType type = 1;
} }
message OwnedVideoStream { message OwnedVideoStream {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
VideoStreamInfo info = 2; required VideoStreamInfo info = 2;
} }
message VideoStreamEvent { message VideoStreamEvent {
uint64 stream_handle = 1; required uint64 stream_handle = 1;
oneof message { oneof message {
VideoFrameReceived frame_received = 2; VideoFrameReceived frame_received = 2;
VideoStreamEOS eos = 3; VideoStreamEOS eos = 3;
@@ -145,9 +159,9 @@ message VideoStreamEvent {
} }
message VideoFrameReceived { message VideoFrameReceived {
OwnedVideoBuffer buffer = 1; required OwnedVideoBuffer buffer = 1;
int64 timestamp_us = 2; // In microseconds required int64 timestamp_us = 2; // In microseconds
VideoRotation rotation = 3; required VideoRotation rotation = 3;
} }
message VideoStreamEOS {} message VideoStreamEOS {}
@@ -157,8 +171,8 @@ message VideoStreamEOS {}
// //
message VideoSourceResolution { message VideoSourceResolution {
uint32 width = 1; required uint32 width = 1;
uint32 height = 2; required uint32 height = 2;
} }
enum VideoSourceType { enum VideoSourceType {
@@ -166,10 +180,10 @@ enum VideoSourceType {
} }
message VideoSourceInfo { message VideoSourceInfo {
VideoSourceType type = 1; required VideoSourceType type = 1;
} }
message OwnedVideoSource { message OwnedVideoSource {
FfiOwnedHandle handle = 1; required FfiOwnedHandle handle = 1;
VideoSourceInfo info = 2; required VideoSourceInfo info = 2;
} }

75
src/qml/CallPage.qml
View File

@@ -12,66 +12,46 @@ Kirigami.Page {
title: i18nc("@title", "Call") title: i18nc("@title", "Call")
VideoOutput { RowLayout {
id: video
anchors.fill: parent anchors.fill: parent
visible: false
Component.onCompleted: CallController.setVideoSink(video.videoSink)
}
VideoOutput { VideoOutput {
id: viewFinder id: viewFinder
anchors.centerIn: parent
ToolBar { Layout.fillWidth: parent.width / 2
id: toolbar Layout.preferredHeight: parent.height
anchors.horizontalCenter: parent.horizontalCenter Label {
anchors.bottom: parent.bottom text: "You"
anchors.bottomMargin: Kirigami.Units.gridUnit * 8
z: 1000
background: Kirigami.ShadowedRectangle {
color: Kirigami.Theme.backgroundColor
radius: 5
shadow {
size: 15
yOffset: 3
color: Qt.rgba(0, 0, 0, 0.2)
}
border {
color: Kirigami.ColorUtils.tintWithAlpha(Kirigami.Theme.backgroundColor, Kirigami.Theme.textColor, 0.2)
width: 1
}
Kirigami.Theme.inherit: false
Kirigami.Theme.colorSet: Kirigami.Theme.Window
} }
RowLayout { }
ToolButton {
id: cameraButton
icon.name: "camera-on-symbolic"
text: i18nc("@action:button", "Enable Camera")
display: AbstractButton.IconOnly
checkable: true
onClicked: CallController.toggleCamera()
ToolTip.text: text VideoOutput {
ToolTip.visible: hovered id: otherViewFinder
ToolTip.delay: Kirigami.Units.toolTipDelay
} Layout.fillWidth: parent.width / 2
Layout.preferredHeight: parent.height
Label {
text: "Them"
} }
} }
} }
LivekitVideoSink { LivekitVideoSink {
videoSink: viewFinder.videoSink videoSink: otherViewFinder.videoSink
} }
//Component.onCompleted: camera.start() Component.onCompleted: camera.start()
Connections {
target: CallController
function onConnected(): void {
CallController.setCameraVideoSink(viewFinder.videoSink)
CallController.toggleCamera()
}
}
CaptureSession { CaptureSession {
camera: Camera { camera: Camera {
@@ -82,5 +62,4 @@ Kirigami.Page {
} }
videoOutput: viewFinder videoOutput: viewFinder
} }
} }