ESCA111/neochat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update to latest LiveKit, fix up QML

Browse Source
This commit is contained in:
Joshua Goins
2024-11-22 08:53:03 -05:00
committed by Tobias Fella
parent ddc16a17d2
commit 2f39e70b67
14 changed files with 1053 additions and 641 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
src/CMakeLists.txt
View File

@@ -411,10 +411,6 @@ if (NOT ANDROID AND NOT WIN32 AND NOT APPLE AND NOT HAIKU)
target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER)
target_compile_definitions(neochat PUBLIC -DHAVE_X11=1)
target_sources(neochat PRIVATE runner.cpp)
if (NOT ANDROID AND NOT WIN32 AND NOT APPLE)
#target_compile_definitions(neochat PUBLIC -DHAVE_RUNNER)
target_compile_definitions(neochat PUBLIC -DHAVE_X11)
#target_sources(neochat PRIVATE runner.cpp)
if (TARGET KUnifiedPush)
target_sources(neochat PRIVATE fakerunner.cpp)
@@ -435,9 +431,10 @@ qt_add_protobuf(neochat
protocols/participant.proto
protocols/stats.proto
protocols/track.proto
protocols/rpc.proto
)
target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums)
target_include_directories(neochat PRIVATE ${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/models ${CMAKE_CURRENT_SOURCE_DIR}/enums ${CMAKE_CURRENT_SOURCE_DIR}/calls)
target_link_libraries(neochat PRIVATE settingsplugin timelineplugin devtoolsplugin loginplugin chatbarplugin)
target_link_libraries(neochat PUBLIC
Qt::Core

32
src/calls/callcontroller.cpp
View File

@@ -11,7 +11,6 @@
#include <QVideoFrame>
#include <QVideoFrameFormat>
#include <QVideoSink>
#include <qprotobufregistration.h>
#include <livekit_ffi.h>
@@ -29,7 +28,7 @@ using namespace livekit::proto;
using namespace Quotient;
extern "C" {
void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs);
void livekit_ffi_initialize(void(ffiCallbackFn(const uint8_t *, size_t)), bool capture_logs, const char *, const char *);
}
void callback(const uint8_t *data, size_t length)
@@ -50,7 +49,7 @@ CallController::CallController()
void CallController::init()
{
qRegisterProtobufTypes();
livekit_ffi_initialize(callback, true);
livekit_ffi_initialize(callback, true, "test", "1.0");
}
static void handleLog(LogRecordRepeated &&logs)
@@ -65,15 +64,16 @@ static void handleLog(LogRecordRepeated &&logs)
void CallController::handleConnect(ConnectCallback &&callback)
{
qWarning() << "Connecting to" << callback.room().info().name() << "with id" << callback.asyncId();
qWarning() << "Connecting to" << callback.result().room().info().name() << "with id" << callback.asyncId();
if (!m_connectingRooms.contains(callback.asyncId()) || !m_connectingRooms[callback.asyncId()]
|| m_connectingRooms[callback.asyncId()]->id() != callback.room().info().name()) {
qWarning() << "Connecting to unexpected room";
return;
}
m_connectingRooms.remove(callback.asyncId());
m_rooms[callback.asyncId()] = callback.room();
localParticipant = callback.localParticipant().handle().id_proto();
m_rooms[callback.asyncId()] = callback.result().room();
localParticipant = callback.result().localParticipant().handle().id_proto();
Q_EMIT connected();
}
void CallController::handleDispose(DisposeCallback &&callback)
@@ -92,7 +92,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event)
if (event.hasParticipantConnected()) {
qWarning() << "Participant connected" << event.participantConnected().info().info().identity();
} else if (event.hasParticipantDisconnected()) {
qWarning() << "Participant connected" << event.participantDisconnected().participantSid();
qWarning() << "Participant connected" << event.participantDisconnected().participantIdentity();
} else if (event.hasLocalTrackPublished()) {
qWarning() << "Local track published";
m_localVideoTrackSid = event.localTrackPublished().trackSid();
@@ -140,7 +140,7 @@ void CallController::handleRoomEvent(livekit::proto::RoomEvent &&event)
} else if (event.hasTrackUnmuted()) {
qWarning() << "Track unmuted";
} else if (event.hasActiveSpeakersChanged()) {
// qWarning() << "Active speakers changed";
qWarning() << "Active speakers changed";
} else if (event.hasRoomMetadataChanged()) {
qWarning() << "room metadata changed";
} else if (event.hasParticipantMetadataChanged()) {
@@ -265,10 +265,6 @@ void CallController::handleEvent(FfiEvent &&event)
qWarning() << "publish data";
} else if (event.hasCaptureAudioFrame()) {
qWarning() << "audio frame";
} else if (event.hasUpdateLocalMetadata()) {
qWarning() << "update local metadata";
} else if (event.hasUpdateLocalName()) {
qWarning() << "update local name";
} else if (event.hasGetStats()) {
qWarning() << "get stats";
} else if (event.hasGetSessionStats()) {
@@ -297,11 +293,11 @@ void CallController::handleCallMemberEvent(const Quotient::CallMemberEvent *even
{"device_id"_ls, connection->deviceId()},
})
.toJson();
if (event->memberships().isEmpty()) {
// This is an old event!
if (!event->contentJson().contains("foci_preferred"_ls)) {
return;
}
auto membership = event->memberships()[0].toObject();
QNetworkRequest request(QUrl((membership["foci_active"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls)));
QNetworkRequest request(QUrl((event->contentJson()["foci_preferred"_ls].toArray()[0]["livekit_service_url"_ls].toString() + "/sfu/get"_ls)));
request.setHeader(QNetworkRequest::ContentTypeHeader, "application/json"_ls);
auto reply = nam->post(request, json);
connect(reply, &QNetworkReply::finished, this, [reply, this, room]() {
@@ -363,6 +359,7 @@ void CallController::setCameraVideoSink(QVideoSink *videoSink)
ffiRequest.setNewVideoSource(newVideoSourceRequest);
auto response = request(std::move(ffiRequest));
handle = response.newVideoSource().source().handle().id_proto();
m_localVideoTrackHandle = handle;
CreateVideoTrackRequest createVideoTrackRequest;
createVideoTrackRequest.setName("Camera"_ls);
@@ -410,7 +407,7 @@ void CallController::setVideoSink(QObject *sink)
void LivekitVideoSink::setVideoSink(QVideoSink *videoSink)
{
m_videoSink = videoSink;
CallController::instance().setCameraVideoSink(videoSink);
CallController::instance().setVideoSink(videoSink);
Q_EMIT videoSinkChanged();
}
QVideoSink *LivekitVideoSink::videoSink() const
@@ -437,10 +434,11 @@ void CallController::publishTrack(uint64_t id)
{
PublishTrackRequest publishTrackRequest;
publishTrackRequest.setTrackHandle(id);
publishTrackRequest.setTrackHandle(m_localVideoTrackHandle);
publishTrackRequest.setLocalParticipantHandle(localParticipant);
TrackPublishOptions options;
options.setSource(TrackSourceGadget::SOURCE_CAMERA);
options.setVideoCodec(VideoCodecGadget::VideoCodec::VP8);
publishTrackRequest.setOptions(options);
auto request = FfiRequest();

4
src/calls/callcontroller.h
View File

@@ -48,12 +48,13 @@ public:
// Internal. Do not use.
void handleEvent(livekit::proto::FfiEvent &&event);
Q_INVOKABLE void setVideoSink(QObject *sink);
void setCameraVideoSink(QVideoSink *videoSink);
Q_INVOKABLE void setCameraVideoSink(QVideoSink *videoSink);
Q_INVOKABLE void toggleCamera();
Q_SIGNALS:
void callStarted();
void connected();
private:
CallController();
@@ -74,6 +75,7 @@ private:
uint64_t localParticipant = 100000;
QString m_localVideoTrackSid;
uint64_t m_localVideoTrackId;
uint64_t m_localVideoTrackHandle;
};
class LivekitVideoSink : public QObject

174
src/protocols/audio_frame.proto
View File

@@ -12,76 +12,162 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
import "handle.proto";
import "track.proto";
// Create a new AudioStream
// AudioStream is used to receive audio frames from a track
message NewAudioStreamRequest {
uint64 track_handle = 1;
AudioStreamType type = 2;
required uint64 track_handle = 1;
required AudioStreamType type = 2;
optional uint32 sample_rate = 3;
optional uint32 num_channels = 4;
}
message NewAudioStreamResponse { OwnedAudioStream stream = 1; }
message NewAudioStreamResponse { required OwnedAudioStream stream = 1; }
message AudioStreamFromParticipantRequest {
required uint64 participant_handle = 1;
required AudioStreamType type = 2;
optional TrackSource track_source = 3;
optional uint32 sample_rate = 5;
optional uint32 num_channels = 6;
}
message AudioStreamFromParticipantResponse { required OwnedAudioStream stream = 1; }
// Create a new AudioSource
message NewAudioSourceRequest {
AudioSourceType type = 1;
required AudioSourceType type = 1;
optional AudioSourceOptions options = 2;
uint32 sample_rate = 3;
uint32 num_channels = 4;
required uint32 sample_rate = 3;
required uint32 num_channels = 4;
optional uint32 queue_size_ms = 5;
}
message NewAudioSourceResponse { OwnedAudioSource source = 1; }
message NewAudioSourceResponse { required OwnedAudioSource source = 1; }
// Push a frame to an AudioSource
// The data provided must be available as long as the client receive the callback.
message CaptureAudioFrameRequest {
uint64 source_handle = 1;
AudioFrameBufferInfo buffer = 2;
required uint64 source_handle = 1;
required AudioFrameBufferInfo buffer = 2;
}
message CaptureAudioFrameResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message CaptureAudioFrameCallback {
uint64 async_id = 1;
required uint64 async_id = 1;
optional string error = 2;
}
message ClearAudioBufferRequest {
required uint64 source_handle = 1;
}
message ClearAudioBufferResponse {}
// Create a new AudioResampler
message NewAudioResamplerRequest {}
message NewAudioResamplerResponse {
OwnedAudioResampler resampler = 1;
required OwnedAudioResampler resampler = 1;
}
// Remix and resample an audio frame
message RemixAndResampleRequest {
uint64 resampler_handle = 1;
AudioFrameBufferInfo buffer = 2;
uint32 num_channels = 3;
uint32 sample_rate = 4;
required uint64 resampler_handle = 1;
required AudioFrameBufferInfo buffer = 2;
required uint32 num_channels = 3;
required uint32 sample_rate = 4;
}
message RemixAndResampleResponse {
OwnedAudioFrameBuffer buffer = 1;
required OwnedAudioFrameBuffer buffer = 1;
}
// New resampler using SoX (much better quality)
message NewSoxResamplerRequest {
required double input_rate = 1;
required double output_rate = 2;
required uint32 num_channels = 3;
required SoxResamplerDataType input_data_type = 4;
required SoxResamplerDataType output_data_type = 5;
required SoxQualityRecipe quality_recipe = 6;
optional uint32 flags = 7;
}
message NewSoxResamplerResponse {
oneof message {
OwnedSoxResampler resampler = 1;
string error = 2;
}
}
message PushSoxResamplerRequest {
required uint64 resampler_handle = 1;
required uint64 data_ptr = 2; // *const i16
required uint32 size = 3; // in bytes
}
message PushSoxResamplerResponse {
required uint64 output_ptr = 1; // *const i16 (could be null)
required uint32 size = 2; // in bytes
optional string error = 3;
}
message FlushSoxResamplerRequest {
required uint64 resampler_handle = 1;
}
message FlushSoxResamplerResponse {
required uint64 output_ptr = 1; // *const i16 (could be null)
required uint32 size = 2; // in bytes
optional string error = 3;
}
enum SoxResamplerDataType {
// TODO(theomonnom): support other datatypes (shouldn't really be needed)
SOXR_DATATYPE_INT16I = 0;
SOXR_DATATYPE_INT16S = 1;
}
enum SoxQualityRecipe {
SOXR_QUALITY_QUICK = 0;
SOXR_QUALITY_LOW = 1;
SOXR_QUALITY_MEDIUM = 2;
SOXR_QUALITY_HIGH = 3;
SOXR_QUALITY_VERYHIGH = 4;
}
enum SoxFlagBits {
SOXR_ROLLOFF_SMALL = 0; // 1 << 0
SOXR_ROLLOFF_MEDIUM = 1; // 1 << 1
SOXR_ROLLOFF_NONE = 2; // 1 << 2
SOXR_HIGH_PREC_CLOCK = 3; // 1 << 3
SOXR_DOUBLE_PRECISION = 4; // 1 << 4
SOXR_VR = 5; // 1 << 5
}
//
// AudioFrame buffer
//
message AudioFrameBufferInfo {
uint64 data_ptr = 1; // *const i16
uint32 num_channels = 2;
uint32 sample_rate = 3;
uint32 samples_per_channel = 4;
required uint64 data_ptr = 1; // *const i16
required uint32 num_channels = 2;
required uint32 sample_rate = 3;
required uint32 samples_per_channel = 4;
}
message OwnedAudioFrameBuffer {
FfiOwnedHandle handle = 1;
AudioFrameBufferInfo info = 2;
required FfiOwnedHandle handle = 1;
required AudioFrameBufferInfo info = 2;
}
//
@@ -94,16 +180,16 @@ enum AudioStreamType {
}
message AudioStreamInfo {
AudioStreamType type = 1;
required AudioStreamType type = 1;
}
message OwnedAudioStream {
FfiOwnedHandle handle = 1;
AudioStreamInfo info = 2;
required FfiOwnedHandle handle = 1;
required AudioStreamInfo info = 2;
}
message AudioStreamEvent {
uint64 stream_handle = 1;
required uint64 stream_handle = 1;
oneof message {
AudioFrameReceived frame_received = 2;
AudioStreamEOS eos = 3;
@@ -111,7 +197,7 @@ message AudioStreamEvent {
}
message AudioFrameReceived {
OwnedAudioFrameBuffer frame = 1;
required OwnedAudioFrameBuffer frame = 1;
}
message AudioStreamEOS {}
@@ -121,9 +207,9 @@ message AudioStreamEOS {}
//
message AudioSourceOptions {
bool echo_cancellation = 1;
bool noise_suppression = 2;
bool auto_gain_control = 3;
required bool echo_cancellation = 1;
required bool noise_suppression = 2;
required bool auto_gain_control = 3;
}
enum AudioSourceType {
@@ -131,12 +217,12 @@ enum AudioSourceType {
}
message AudioSourceInfo {
AudioSourceType type = 2;
required AudioSourceType type = 2;
}
message OwnedAudioSource {
FfiOwnedHandle handle = 1;
AudioSourceInfo info = 2;
required FfiOwnedHandle handle = 1;
required AudioSourceInfo info = 2;
}
//
@@ -146,6 +232,20 @@ message OwnedAudioSource {
message AudioResamplerInfo { }
message OwnedAudioResampler {
FfiOwnedHandle handle = 1;
AudioResamplerInfo info = 2;
required FfiOwnedHandle handle = 1;
required AudioResamplerInfo info = 2;
}
//
// Sox AudioResampler
//
message SoxResamplerInfo {}
message OwnedSoxResampler {
required FfiOwnedHandle handle = 1;
required SoxResamplerInfo info = 2;
}

64
src/protocols/e2ee.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -26,23 +26,23 @@ enum EncryptionType {
}
message FrameCryptor {
string participant_identity = 1;
string track_sid = 2;
int32 key_index = 3;
bool enabled = 4;
required string participant_identity = 1;
required string track_sid = 2;
required int32 key_index = 3;
required bool enabled = 4;
}
message KeyProviderOptions {
// Only specify if you want to use a shared_key
optional bytes shared_key = 1;
int32 ratchet_window_size = 2;
bytes ratchet_salt = 3;
int32 failure_tolerance = 4; // -1 = no tolerence
required int32 ratchet_window_size = 2;
required bytes ratchet_salt = 3;
required int32 failure_tolerance = 4; // -1 = no tolerance
}
message E2eeOptions {
EncryptionType encryption_type = 1;
KeyProviderOptions key_provider_options = 2;
required EncryptionType encryption_type = 1;
required KeyProviderOptions key_provider_options = 2;
}
enum EncryptionState {
@@ -56,7 +56,7 @@ enum EncryptionState {
}
message E2eeManagerSetEnabledRequest {
bool enabled = 1;
required bool enabled = 1;
}
message E2eeManagerSetEnabledResponse {}
@@ -66,64 +66,64 @@ message E2eeManagerGetFrameCryptorsResponse {
}
message FrameCryptorSetEnabledRequest {
string participant_identity = 1;
string track_sid = 2;
bool enabled = 3;
required string participant_identity = 1;
required string track_sid = 2;
required bool enabled = 3;
}
message FrameCryptorSetEnabledResponse { }
message FrameCryptorSetEnabledResponse {}
message FrameCryptorSetKeyIndexRequest {
string participant_identity = 1;
string track_sid = 2;
int32 key_index = 3;
required string participant_identity = 1;
required string track_sid = 2;
required int32 key_index = 3;
}
message FrameCryptorSetKeyIndexResponse { }
message FrameCryptorSetKeyIndexResponse {}
message SetSharedKeyRequest {
bytes shared_key = 1;
int32 key_index = 2;
required bytes shared_key = 1;
required int32 key_index = 2;
}
message SetSharedKeyResponse { }
message SetSharedKeyResponse {}
message RatchetSharedKeyRequest {
int32 key_index = 1;
required int32 key_index = 1;
}
message RatchetSharedKeyResponse {
optional bytes new_key = 1;
}
message GetSharedKeyRequest {
int32 key_index = 1;
required int32 key_index = 1;
}
message GetSharedKeyResponse {
optional bytes key = 1;
}
message SetKeyRequest {
string participant_identity = 1;
bytes key = 2;
int32 key_index = 3;
required string participant_identity = 1;
required bytes key = 2;
required int32 key_index = 3;
}
message SetKeyResponse {}
message RatchetKeyRequest {
string participant_identity = 1;
int32 key_index = 2;
required string participant_identity = 1;
required int32 key_index = 2;
}
message RatchetKeyResponse {
optional bytes new_key = 1;
}
message GetKeyRequest {
string participant_identity = 1;
int32 key_index = 2;
required string participant_identity = 1;
required int32 key_index = 2;
}
message GetKeyResponse {
optional bytes key = 1;
}
message E2eeRequest {
uint64 room_handle = 1;
required uint64 room_handle = 1;
oneof message {
E2eeManagerSetEnabledRequest manager_set_enabled = 2;
E2eeManagerGetFrameCryptorsRequest manager_get_frame_cryptors = 3;

148
src/protocols/ffi.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -23,6 +23,7 @@ import "track.proto";
import "room.proto";
import "video_frame.proto";
import "audio_frame.proto";
import "rpc.proto";
// **How is the livekit-ffi working:
// We refer as the ffi server the Rust server that is running the LiveKit client implementation, and we
@@ -63,28 +64,48 @@ message FfiRequest {
UnpublishTrackRequest unpublish_track = 6;
PublishDataRequest publish_data = 7;
SetSubscribedRequest set_subscribed = 8;
UpdateLocalMetadataRequest update_local_metadata = 9;
UpdateLocalNameRequest update_local_name = 10;
GetSessionStatsRequest get_session_stats = 11;
SetLocalMetadataRequest set_local_metadata = 9;
SetLocalNameRequest set_local_name = 10;
SetLocalAttributesRequest set_local_attributes = 11;
GetSessionStatsRequest get_session_stats = 12;
PublishTranscriptionRequest publish_transcription = 13;
PublishSipDtmfRequest publish_sip_dtmf = 14;
// Track
CreateVideoTrackRequest create_video_track = 12;
CreateAudioTrackRequest create_audio_track = 13;
GetStatsRequest get_stats = 14;
CreateVideoTrackRequest create_video_track = 15;
CreateAudioTrackRequest create_audio_track = 16;
LocalTrackMuteRequest local_track_mute = 17;
EnableRemoteTrackRequest enable_remote_track = 18;
GetStatsRequest get_stats = 19;
// Video
NewVideoStreamRequest new_video_stream = 16;
NewVideoSourceRequest new_video_source = 17;
CaptureVideoFrameRequest capture_video_frame = 18;
VideoConvertRequest video_convert = 19;
// Video
NewVideoStreamRequest new_video_stream = 20;
NewVideoSourceRequest new_video_source = 21;
CaptureVideoFrameRequest capture_video_frame = 22;
VideoConvertRequest video_convert = 23;
VideoStreamFromParticipantRequest video_stream_from_participant = 24;
// Audio
NewAudioStreamRequest new_audio_stream = 22;
NewAudioSourceRequest new_audio_source = 23;
CaptureAudioFrameRequest capture_audio_frame = 24;
NewAudioResamplerRequest new_audio_resampler = 25;
RemixAndResampleRequest remix_and_resample = 26;
E2eeRequest e2ee = 27;
NewAudioStreamRequest new_audio_stream = 25;
NewAudioSourceRequest new_audio_source = 26;
CaptureAudioFrameRequest capture_audio_frame = 27;
ClearAudioBufferRequest clear_audio_buffer = 28;
NewAudioResamplerRequest new_audio_resampler = 29;
RemixAndResampleRequest remix_and_resample = 30;
E2eeRequest e2ee = 31;
AudioStreamFromParticipantRequest audio_stream_from_participant = 32;
NewSoxResamplerRequest new_sox_resampler = 33;
PushSoxResamplerRequest push_sox_resampler = 34;
FlushSoxResamplerRequest flush_sox_resampler = 35;
SendChatMessageRequest send_chat_message = 36;
EditChatMessageRequest edit_chat_message = 37;
// RPC
PerformRpcRequest perform_rpc = 38;
RegisterRpcMethodRequest register_rpc_method = 39;
UnregisterRpcMethodRequest unregister_rpc_method = 40;
RpcMethodInvocationResponseRequest rpc_method_invocation_response = 41;
}
}
@@ -100,28 +121,45 @@ message FfiResponse {
UnpublishTrackResponse unpublish_track = 6;
PublishDataResponse publish_data = 7;
SetSubscribedResponse set_subscribed = 8;
UpdateLocalMetadataResponse update_local_metadata = 9;
UpdateLocalNameResponse update_local_name = 10;
GetSessionStatsResponse get_session_stats = 11;
SetLocalMetadataResponse set_local_metadata = 9;
SetLocalNameResponse set_local_name = 10;
SetLocalAttributesResponse set_local_attributes = 11;
GetSessionStatsResponse get_session_stats = 12;
PublishTranscriptionResponse publish_transcription = 13;
PublishSipDtmfResponse publish_sip_dtmf = 14;
// Track
CreateVideoTrackResponse create_video_track = 12;
CreateAudioTrackResponse create_audio_track = 13;
GetStatsResponse get_stats = 14;
CreateVideoTrackResponse create_video_track = 15;
CreateAudioTrackResponse create_audio_track = 16;
LocalTrackMuteResponse local_track_mute = 17;
EnableRemoteTrackResponse enable_remote_track = 18;
GetStatsResponse get_stats = 19;
// Video
NewVideoStreamResponse new_video_stream = 16;
NewVideoSourceResponse new_video_source = 17;
CaptureVideoFrameResponse capture_video_frame = 18;
VideoConvertResponse video_convert = 19;
NewVideoStreamResponse new_video_stream = 20;
NewVideoSourceResponse new_video_source = 21;
CaptureVideoFrameResponse capture_video_frame = 22;
VideoConvertResponse video_convert = 23;
VideoStreamFromParticipantResponse video_stream_from_participant = 24;
// Audio
NewAudioStreamResponse new_audio_stream = 22;
NewAudioSourceResponse new_audio_source = 23;
CaptureAudioFrameResponse capture_audio_frame = 24;
NewAudioResamplerResponse new_audio_resampler = 25;
RemixAndResampleResponse remix_and_resample = 26;
E2eeResponse e2ee = 27;
NewAudioStreamResponse new_audio_stream = 25;
NewAudioSourceResponse new_audio_source = 26;
CaptureAudioFrameResponse capture_audio_frame = 27;
ClearAudioBufferResponse clear_audio_buffer = 28;
NewAudioResamplerResponse new_audio_resampler = 29;
RemixAndResampleResponse remix_and_resample = 30;
AudioStreamFromParticipantResponse audio_stream_from_participant = 31;
E2eeResponse e2ee = 32;
NewSoxResamplerResponse new_sox_resampler = 33;
PushSoxResamplerResponse push_sox_resampler = 34;
FlushSoxResamplerResponse flush_sox_resampler = 35;
SendChatMessageResponse send_chat_message = 36;
// RPC
PerformRpcResponse perform_rpc = 37;
RegisterRpcMethodResponse register_rpc_method = 38;
UnregisterRpcMethodResponse unregister_rpc_method = 39;
RpcMethodInvocationResponseResponse rpc_method_invocation_response = 40;
}
}
@@ -135,18 +173,24 @@ message FfiEvent {
VideoStreamEvent video_stream_event = 3;
AudioStreamEvent audio_stream_event = 4;
ConnectCallback connect = 5;
DisconnectCallback disconnect = 6;
DisposeCallback dispose = 7;
PublishTrackCallback publish_track = 8;
UnpublishTrackCallback unpublish_track = 9;
PublishDataCallback publish_data = 10;
CaptureAudioFrameCallback capture_audio_frame = 11;
UpdateLocalMetadataCallback update_local_metadata = 12;
UpdateLocalNameCallback update_local_name = 13;
GetStatsCallback get_stats = 14;
LogBatch logs = 15;
GetSessionStatsCallback get_session_stats = 16;
Panic panic = 17;
DisconnectCallback disconnect = 7;
DisposeCallback dispose = 8;
PublishTrackCallback publish_track = 9;
UnpublishTrackCallback unpublish_track = 10;
PublishDataCallback publish_data = 11;
PublishTranscriptionCallback publish_transcription = 12;
CaptureAudioFrameCallback capture_audio_frame = 13;
SetLocalMetadataCallback set_local_metadata = 14;
SetLocalNameCallback set_local_name = 15;
SetLocalAttributesCallback set_local_attributes = 16;
GetStatsCallback get_stats = 17;
LogBatch logs = 18;
GetSessionStatsCallback get_session_stats = 19;
Panic panic = 20;
PublishSipDtmfCallback publish_sip_dtmf = 21;
SendChatMessageCallback chat_message = 22;
PerformRpcCallback perform_rpc = 23;
RpcMethodInvocationEvent rpc_method_invocation = 24;
}
}
@@ -154,14 +198,14 @@ message FfiEvent {
// e.g: This is used for the Unity Editor after each assemblies reload.
// TODO(theomonnom): Implement a debug mode where we can find all leaked handles?
message DisposeRequest {
bool async = 1;
required bool async = 1;
}
message DisposeResponse {
optional uint64 async_id = 1; // None if sync
}
message DisposeCallback {
uint64 async_id = 1;
required uint64 async_id = 1;
}
enum LogLevel {
@@ -173,12 +217,12 @@ enum LogLevel {
}
message LogRecord {
LogLevel level = 1;
string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc...
required LogLevel level = 1;
required string target = 2; // e.g "livekit", "libwebrtc", "tokio-tungstenite", etc...
optional string module_path = 3;
optional string file = 4;
optional uint32 line = 5;
string message = 6;
required string message = 6;
}
message LogBatch {
@@ -186,7 +230,7 @@ message LogBatch {
}
message Panic {
string message = 1;
required string message = 1;
}
// TODO(theomonnom): Debug messages (Print handles).

4
src/protocols/handle.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -27,5 +27,5 @@ option csharp_namespace = "LiveKit.Proto";
// When refering to a handle without owning it, we just use a uint32 without this message.
// (the variable name is suffixed with "_handle")
message FfiOwnedHandle {
uint64 id = 1;
required uint64 id = 1;
}

24
src/protocols/participant.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -20,13 +20,23 @@ option csharp_namespace = "LiveKit.Proto";
import "handle.proto";
message ParticipantInfo {
string sid = 1;
string name = 2;
string identity = 3;
string metadata = 4;
required string sid = 1;
required string name = 2;
required string identity = 3;
required string metadata = 4;
map<string, string> attributes = 5;
required ParticipantKind kind = 6;
}
message OwnedParticipant {
FfiOwnedHandle handle = 1;
ParticipantInfo info = 2;
required FfiOwnedHandle handle = 1;
required ParticipantInfo info = 2;
}
enum ParticipantKind {
PARTICIPANT_KIND_STANDARD = 0;
PARTICIPANT_KIND_INGRESS = 1;
PARTICIPANT_KIND_EGRESS = 2;
PARTICIPANT_KIND_SIP = 3;
PARTICIPANT_KIND_AGENT = 4;
}

443
src/protocols/room.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -26,147 +26,240 @@ import "stats.proto";
// Connect to a new LiveKit room
message ConnectRequest {
string url = 1;
string token = 2;
RoomOptions options = 3;
required string url = 1;
required string token = 2;
required RoomOptions options = 3;
}
message ConnectResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message ConnectCallback {
message ParticipantWithTracks {
OwnedParticipant participant = 1;
required OwnedParticipant participant = 1;
// TrackInfo are not needed here, if we're subscribed to a track, the FfiServer will send
// a TrackSubscribed event
repeated OwnedTrackPublication publications = 2;
}
uint64 async_id = 1;
optional string error = 2;
OwnedRoom room = 3;
OwnedParticipant local_participant = 4;
repeated ParticipantWithTracks participants = 5;
message Result {
required OwnedRoom room = 1;
required OwnedParticipant local_participant = 2;
repeated ParticipantWithTracks participants = 3;
}
required uint64 async_id = 1;
oneof message {
string error = 2;
Result result = 3;
}
}
// Disconnect from the a room
message DisconnectRequest { uint64 room_handle = 1; }
message DisconnectResponse { uint64 async_id = 1; }
message DisconnectCallback { uint64 async_id = 1; }
message DisconnectRequest { required uint64 room_handle = 1; }
message DisconnectResponse { required uint64 async_id = 1; }
message DisconnectCallback { required uint64 async_id = 1; }
// Publish a track to the room
message PublishTrackRequest {
uint64 local_participant_handle = 1;
uint64 track_handle = 2;
TrackPublishOptions options = 3;
required uint64 local_participant_handle = 1;
required uint64 track_handle = 2;
required TrackPublishOptions options = 3;
}
message PublishTrackResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message PublishTrackCallback {
uint64 async_id = 1;
optional string error = 2;
OwnedTrackPublication publication = 3;
required uint64 async_id = 1;
oneof message {
string error = 2;
OwnedTrackPublication publication = 3;
}
}
// Unpublish a track from the room
message UnpublishTrackRequest {
uint64 local_participant_handle = 1;
string track_sid = 2;
bool stop_on_unpublish = 3;
required uint64 local_participant_handle = 1;
required string track_sid = 2;
required bool stop_on_unpublish = 3;
}
message UnpublishTrackResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message UnpublishTrackCallback {
uint64 async_id = 1;
required uint64 async_id = 1;
optional string error = 2;
}
// Publish data to other participants
message PublishDataRequest {
uint64 local_participant_handle = 1;
uint64 data_ptr = 2;
uint64 data_len = 3;
DataPacketKind kind = 4;
repeated string destination_sids = 5; // destination
required uint64 local_participant_handle = 1;
required uint64 data_ptr = 2;
required uint64 data_len = 3;
required bool reliable = 4;
repeated string destination_sids = 5 [deprecated=true];
optional string topic = 6;
repeated string destination_identities = 7;
}
message PublishDataResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message PublishDataCallback {
uint64 async_id = 1;
required uint64 async_id = 1;
optional string error = 2;
}
// Publish transcription messages to room
message PublishTranscriptionRequest {
required uint64 local_participant_handle = 1;
required string participant_identity = 2;
required string track_id = 3;
repeated TranscriptionSegment segments = 4;
}
message PublishTranscriptionResponse {
required uint64 async_id = 1;
}
message PublishTranscriptionCallback {
required uint64 async_id = 1;
optional string error = 2;
}
// Publish Sip DTMF messages to other participants
message PublishSipDtmfRequest {
required uint64 local_participant_handle = 1;
required uint32 code = 2;
required string digit = 3;
repeated string destination_identities = 4;
}
message PublishSipDtmfResponse {
required uint64 async_id = 1;
}
message PublishSipDtmfCallback {
required uint64 async_id = 1;
optional string error = 2;
}
// Change the local participant's metadata
message UpdateLocalMetadataRequest {
uint64 local_participant_handle = 1;
string metadata = 2;
message SetLocalMetadataRequest {
required uint64 local_participant_handle = 1;
required string metadata = 2;
}
message UpdateLocalMetadataResponse {
uint64 async_id = 1;
message SetLocalMetadataResponse {
required uint64 async_id = 1;
}
message UpdateLocalMetadataCallback {
uint64 async_id = 1;
message SetLocalMetadataCallback {
required uint64 async_id = 1;
optional string error = 2;
}
message SendChatMessageRequest {
required uint64 local_participant_handle = 1;
required string message = 2;
repeated string destination_identities = 3;
optional string sender_identity = 4;
}
message EditChatMessageRequest {
required uint64 local_participant_handle = 1;
required string edit_text = 2;
required ChatMessage original_message = 3;
repeated string destination_identities = 4;
optional string sender_identity = 5;
}
message SendChatMessageResponse {
required uint64 async_id = 1;
}
message SendChatMessageCallback {
required uint64 async_id = 1;
oneof message {
string error = 2;
ChatMessage chat_message = 3;
}
}
// Change the local participant's attributes
message SetLocalAttributesRequest {
required uint64 local_participant_handle = 1;
repeated AttributesEntry attributes = 2;
}
message AttributesEntry {
required string key = 1;
required string value = 2;
}
message SetLocalAttributesResponse {
required uint64 async_id = 1;
}
message SetLocalAttributesCallback {
required uint64 async_id = 1;
optional string error = 2;
}
// Change the local participant's name
message UpdateLocalNameRequest {
uint64 local_participant_handle = 1;
string name = 2;
message SetLocalNameRequest {
required uint64 local_participant_handle = 1;
required string name = 2;
}
message UpdateLocalNameResponse {
uint64 async_id = 1;
message SetLocalNameResponse {
required uint64 async_id = 1;
}
message UpdateLocalNameCallback {
uint64 async_id = 1;
message SetLocalNameCallback {
required uint64 async_id = 1;
optional string error = 2;
}
// Change the "desire" to subs2ribe to a track
message SetSubscribedRequest {
bool subscribe = 1;
uint64 publication_handle = 2;
required bool subscribe = 1;
required uint64 publication_handle = 2;
}
message SetSubscribedResponse {}
message GetSessionStatsRequest {
uint64 room_handle = 1;
required uint64 room_handle = 1;
}
message GetSessionStatsResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message GetSessionStatsCallback {
uint64 async_id = 1;
optional string error = 2;
repeated RtcStats publisher_stats = 3;
repeated RtcStats subscriber_stats = 4;
}
message Result {
repeated RtcStats publisher_stats = 1;
repeated RtcStats subscriber_stats = 2;
}
required uint64 async_id = 1;
oneof message {
string error = 2;
Result result = 3;
}
}
//
// Options
//
message VideoEncoding {
uint64 max_bitrate = 1;
double max_framerate = 2;
required uint64 max_bitrate = 1;
required double max_framerate = 2;
}
message AudioEncoding {
uint64 max_bitrate = 1;
required uint64 max_bitrate = 1;
}
message TrackPublishOptions {
// encodings are optional
VideoEncoding video_encoding = 1;
AudioEncoding audio_encoding = 2;
VideoCodec video_codec = 3;
bool dtx = 4;
bool red = 5;
bool simulcast = 6;
TrackSource source = 7;
optional VideoEncoding video_encoding = 1;
optional AudioEncoding audio_encoding = 2;
optional VideoCodec video_codec = 3;
optional bool dtx = 4;
optional bool red = 5;
optional bool simulcast = 6;
optional TrackSource source = 7;
optional string stream = 8;
}
enum IceTransportType {
@@ -182,8 +275,8 @@ enum ContinualGatheringPolicy {
message IceServer {
repeated string urls = 1;
string username = 2;
string password = 3;
optional string username = 2;
optional string password = 3;
}
message RtcConfig {
@@ -193,12 +286,12 @@ message RtcConfig {
}
message RoomOptions {
bool auto_subscribe = 1;
bool adaptive_stream = 2;
bool dynacast = 3;
optional bool auto_subscribe = 1;
optional bool adaptive_stream = 2;
optional bool dynacast = 3;
optional E2eeOptions e2ee = 4;
optional RtcConfig rtc_config = 5; // allow to setup a custom RtcConfiguration
uint32 join_retries = 6;
optional uint32 join_retries = 6;
}
//
@@ -223,163 +316,237 @@ enum DataPacketKind {
KIND_RELIABLE = 1;
}
enum DisconnectReason {
UNKNOWN_REASON = 0;
// the client initiated the disconnect
CLIENT_INITIATED = 1;
// another participant with the same identity has joined the room
DUPLICATE_IDENTITY = 2;
// the server instance is shutting down
SERVER_SHUTDOWN = 3;
// RoomService.RemoveParticipant was called
PARTICIPANT_REMOVED = 4;
// RoomService.DeleteRoom was called
ROOM_DELETED = 5;
// the client is attempting to resume a session, but server is not aware of it
STATE_MISMATCH = 6;
// client was unable to connect fully
JOIN_FAILURE = 7;
// Cloud-only, the server requested Participant to migrate the connection elsewhere
MIGRATION = 8;
// the signal websocket was closed unexpectedly
SIGNAL_CLOSE = 9;
// the room was closed, due to all Standard and Ingress participants having left
ROOM_CLOSED = 10;
}
message TranscriptionSegment {
required string id = 1;
required string text = 2;
required uint64 start_time = 3;
required uint64 end_time = 4;
required bool final = 5;
required string language = 6;
}
message BufferInfo {
uint64 data_ptr = 1;
uint64 data_len = 2;
required uint64 data_ptr = 1;
required uint64 data_len = 2;
}
message OwnedBuffer {
FfiOwnedHandle handle = 1;
BufferInfo data = 2;
required FfiOwnedHandle handle = 1;
required BufferInfo data = 2;
}
message RoomEvent {
uint64 room_handle = 1;
required uint64 room_handle = 1;
oneof message {
ParticipantConnected participant_connected = 2;
ParticipantDisconnected participant_disconnected = 3;
LocalTrackPublished local_track_published = 4;
LocalTrackUnpublished local_track_unpublished = 5;
TrackPublished track_published = 6;
TrackUnpublished track_unpublished = 7;
TrackSubscribed track_subscribed = 8;
TrackUnsubscribed track_unsubscribed = 9;
TrackSubscriptionFailed track_subscription_failed = 10;
TrackMuted track_muted = 11;
TrackUnmuted track_unmuted = 12;
ActiveSpeakersChanged active_speakers_changed = 13;
RoomMetadataChanged room_metadata_changed = 14;
ParticipantMetadataChanged participant_metadata_changed = 15;
ParticipantNameChanged participant_name_changed = 16;
ConnectionQualityChanged connection_quality_changed = 17;
ConnectionStateChanged connection_state_changed = 19;
// Connected connected = 20;
Disconnected disconnected = 21;
Reconnecting reconnecting = 22;
Reconnected reconnected = 23;
E2eeStateChanged e2ee_state_changed = 24;
RoomEOS eos = 25; // The stream of room events has ended
DataPacketReceived data_packet_received = 26;
LocalTrackSubscribed local_track_subscribed = 6;
TrackPublished track_published = 7;
TrackUnpublished track_unpublished = 8;
TrackSubscribed track_subscribed = 9;
TrackUnsubscribed track_unsubscribed = 10;
TrackSubscriptionFailed track_subscription_failed = 11;
TrackMuted track_muted = 12;
TrackUnmuted track_unmuted = 13;
ActiveSpeakersChanged active_speakers_changed = 14;
RoomMetadataChanged room_metadata_changed = 15;
RoomSidChanged room_sid_changed = 16;
ParticipantMetadataChanged participant_metadata_changed = 17;
ParticipantNameChanged participant_name_changed = 18;
ParticipantAttributesChanged participant_attributes_changed = 19;
ConnectionQualityChanged connection_quality_changed = 20;
ConnectionStateChanged connection_state_changed = 21;
// Connected connected = 21;
Disconnected disconnected = 22;
Reconnecting reconnecting = 23;
Reconnected reconnected = 24;
E2eeStateChanged e2ee_state_changed = 25;
RoomEOS eos = 26; // The stream of room events has ended
DataPacketReceived data_packet_received = 27;
TranscriptionReceived transcription_received = 28;
ChatMessageReceived chat_message = 29;
}
}
message RoomInfo {
string sid = 1;
string name = 2;
string metadata = 3;
optional string sid = 1;
required string name = 2;
required string metadata = 3;
}
message OwnedRoom {
FfiOwnedHandle handle = 1;
RoomInfo info = 2;
required FfiOwnedHandle handle = 1;
required RoomInfo info = 2;
}
message ParticipantConnected { OwnedParticipant info = 1; }
message ParticipantConnected { required OwnedParticipant info = 1; }
message ParticipantDisconnected {
string participant_sid = 1;
required string participant_identity = 1;
}
message LocalTrackPublished {
// The TrackPublicationInfo comes from the PublishTrack response
// and the FfiClient musts wait for it before firing this event
string track_sid = 1;
required string track_sid = 1;
}
message LocalTrackUnpublished {
string publication_sid = 1;
required string publication_sid = 1;
}
message LocalTrackSubscribed {
required string track_sid = 2;
}
message TrackPublished {
string participant_sid = 1;
OwnedTrackPublication publication = 2;
required string participant_identity = 1;
required OwnedTrackPublication publication = 2;
}
message TrackUnpublished {
string participant_sid = 1;
string publication_sid = 2;
required string participant_identity = 1;
required string publication_sid = 2;
}
// Publication isn't needed for subscription events on the FFI
// The FFI will retrieve the publication using the Track sid
message TrackSubscribed {
string participant_sid = 1;
OwnedTrack track = 2;
required string participant_identity = 1;
required OwnedTrack track = 2;
}
message TrackUnsubscribed {
// The FFI language can dispose/remove the VideoSink here
string participant_sid = 1;
string track_sid = 2;
required string participant_identity = 1;
required string track_sid = 2;
}
message TrackSubscriptionFailed {
string participant_sid = 1;
string track_sid = 2;
string error = 3;
required string participant_identity = 1;
required string track_sid = 2;
required string error = 3;
}
message TrackMuted {
string participant_sid = 1;
string track_sid = 2;
required string participant_identity = 1;
required string track_sid = 2;
}
message TrackUnmuted {
string participant_sid = 1;
string track_sid = 2;
required string participant_identity = 1;
required string track_sid = 2;
}
message E2eeStateChanged {
string participant_sid = 1; // Using sid instead of identity for ffi communication
EncryptionState state = 2;
required string participant_identity = 1; // Using sid instead of identity for ffi communication
required EncryptionState state = 2;
}
message ActiveSpeakersChanged { repeated string participant_sids = 1; }
message ActiveSpeakersChanged { repeated string participant_identities = 1; }
message RoomMetadataChanged {
string metadata = 1;
required string metadata = 1;
}
message RoomSidChanged {
required string sid = 1;
}
message ParticipantMetadataChanged {
string participant_sid = 1;
string metadata = 2;
required string participant_identity = 1;
required string metadata = 2;
}
message ParticipantAttributesChanged {
required string participant_identity = 1;
repeated AttributesEntry attributes = 2;
repeated AttributesEntry changed_attributes = 3;
}
message ParticipantNameChanged {
string participant_sid = 1;
string name = 2;
required string participant_identity = 1;
required string name = 2;
}
message ConnectionQualityChanged {
string participant_sid = 1;
ConnectionQuality quality = 2;
required string participant_identity = 1;
required ConnectionQuality quality = 2;
}
message UserPacket {
OwnedBuffer data = 1;
required OwnedBuffer data = 1;
optional string topic = 2;
}
message ChatMessage {
required string id = 1;
required int64 timestamp = 2;
required string message = 3;
optional int64 edit_timestamp = 4;
optional bool deleted = 5;
optional bool generated = 6;
}
message ChatMessageReceived {
required ChatMessage message = 1;
required string participant_identity = 2;
}
message SipDTMF {
uint32 code = 1;
required uint32 code = 1;
optional string digit = 2;
}
message DataPacketReceived {
DataPacketKind kind = 1;
string participant_identity = 2; // Can be empty if the data is sent a server SDK
optional string participant_sid = 3 [deprecated=true]; // Can be empty if the data is sent a server SDK
required DataPacketKind kind = 1;
required string participant_identity = 2; // Can be empty if the data is sent a server SDK
oneof value {
UserPacket user = 4;
SipDTMF sip_dtmf = 5;
}
}
message ConnectionStateChanged { ConnectionState state = 1; }
message TranscriptionReceived {
optional string participant_identity = 1;
optional string track_sid = 2;
repeated TranscriptionSegment segments = 3;
}
message ConnectionStateChanged { required ConnectionState state = 1; }
message Connected {}
message Disconnected {}
message Disconnected {
required DisconnectReason reason = 1;
}
message Reconnecting {}
message Reconnected {}
message RoomEOS {}

81
src/protocols/rpc.proto Normal file
View File

@@ -0,0 +1,81 @@
// Copyright 2023 LiveKit, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
message RpcError {
required uint32 code = 1;
required string message = 2;
optional string data = 3;
}
// FFI Requests
message PerformRpcRequest {
required uint64 local_participant_handle = 1;
required string destination_identity = 2;
required string method = 3;
required string payload = 4;
optional uint32 response_timeout_ms = 5;
}
message RegisterRpcMethodRequest {
required uint64 local_participant_handle = 1;
required string method = 2;
}
message UnregisterRpcMethodRequest {
required uint64 local_participant_handle = 1;
required string method = 2;
}
message RpcMethodInvocationResponseRequest {
required uint64 local_participant_handle = 1;
required uint64 invocation_id = 2;
optional string payload = 3;
optional RpcError error = 4;
}
// FFI Responses
message PerformRpcResponse {
required uint64 async_id = 1;
}
message RegisterRpcMethodResponse {}
message UnregisterRpcMethodResponse {}
message RpcMethodInvocationResponseResponse {
optional string error = 1;
}
// FFI Callbacks
message PerformRpcCallback {
required uint64 async_id = 1;
optional string payload = 2;
optional RpcError error = 3;
}
// FFI Events
message RpcMethodInvocationEvent {
required uint64 local_participant_handle = 1;
required uint64 invocation_id = 2;
required string method = 3;
required string request_id = 4;
required string caller_identity = 5;
required string payload = 6;
required uint32 response_timeout_ms = 7;
}

456
src/protocols/stats.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -91,83 +91,83 @@ enum IceTcpCandidateType {
message RtcStats {
message Codec {
RtcStatsData rtc = 1;
CodecStats codec = 2;
required RtcStatsData rtc = 1;
required CodecStats codec = 2;
}
message InboundRtp {
RtcStatsData rtc = 1;
RtpStreamStats stream = 2;
ReceivedRtpStreamStats received = 3;
InboundRtpStreamStats inbound = 4;
required RtcStatsData rtc = 1;
required RtpStreamStats stream = 2;
required ReceivedRtpStreamStats received = 3;
required InboundRtpStreamStats inbound = 4;
}
message OutboundRtp {
RtcStatsData rtc = 1;
RtpStreamStats stream = 2;
SentRtpStreamStats sent = 3;
OutboundRtpStreamStats outbound = 4;
required RtcStatsData rtc = 1;
required RtpStreamStats stream = 2;
required SentRtpStreamStats sent = 3;
required OutboundRtpStreamStats outbound = 4;
}
message RemoteInboundRtp {
RtcStatsData rtc = 1;
RtpStreamStats stream = 2;
ReceivedRtpStreamStats received = 3;
RemoteInboundRtpStreamStats remote_inbound = 4;
required RtcStatsData rtc = 1;
required RtpStreamStats stream = 2;
required ReceivedRtpStreamStats received = 3;
required RemoteInboundRtpStreamStats remote_inbound = 4;
}
message RemoteOutboundRtp {
RtcStatsData rtc = 1;
RtpStreamStats stream = 2;
SentRtpStreamStats sent = 3;
RemoteOutboundRtpStreamStats remote_outbound = 4;
required RtcStatsData rtc = 1;
required RtpStreamStats stream = 2;
required SentRtpStreamStats sent = 3;
required RemoteOutboundRtpStreamStats remote_outbound = 4;
}
message MediaSource {
RtcStatsData rtc = 1;
MediaSourceStats source = 2;
AudioSourceStats audio = 3;
VideoSourceStats video = 4;
required RtcStatsData rtc = 1;
required MediaSourceStats source = 2;
required AudioSourceStats audio = 3;
required VideoSourceStats video = 4;
}
message MediaPlayout {
RtcStatsData rtc = 1;
AudioPlayoutStats audio_playout = 2;
required RtcStatsData rtc = 1;
required AudioPlayoutStats audio_playout = 2;
}
message PeerConnection {
RtcStatsData rtc = 1;
PeerConnectionStats pc = 2;
required RtcStatsData rtc = 1;
required PeerConnectionStats pc = 2;
}
message DataChannel {
RtcStatsData rtc = 1;
DataChannelStats dc = 2;
required RtcStatsData rtc = 1;
required DataChannelStats dc = 2;
}
message Transport {
RtcStatsData rtc = 1;
TransportStats transport = 2;
required RtcStatsData rtc = 1;
required TransportStats transport = 2;
}
message CandidatePair {
RtcStatsData rtc = 1;
CandidatePairStats candidate_pair = 2;
required RtcStatsData rtc = 1;
required CandidatePairStats candidate_pair = 2;
}
message LocalCandidate {
RtcStatsData rtc = 1;
IceCandidateStats candidate = 2;
required RtcStatsData rtc = 1;
required IceCandidateStats candidate = 2;
}
message RemoteCandidate {
RtcStatsData rtc = 1;
IceCandidateStats candidate = 2;
required RtcStatsData rtc = 1;
required IceCandidateStats candidate = 2;
}
message Certificate {
RtcStatsData rtc = 1;
CertificateStats certificate = 2;
required RtcStatsData rtc = 1;
required CertificateStats certificate = 2;
}
message Track {
@@ -194,256 +194,256 @@ message RtcStats {
}
message RtcStatsData {
string id = 1;
int64 timestamp = 2;
required string id = 1;
required int64 timestamp = 2;
}
message CodecStats {
uint32 payload_type = 1;
string transport_id = 2;
string mime_type = 3;
uint32 clock_rate = 4;
uint32 channels = 5;
string sdp_fmtp_line = 6;
required uint32 payload_type = 1;
required string transport_id = 2;
required string mime_type = 3;
required uint32 clock_rate = 4;
required uint32 channels = 5;
required string sdp_fmtp_line = 6;
}
message RtpStreamStats {
uint32 ssrc = 1;
string kind = 2;
string transport_id = 3;
string codec_id = 4;
required uint32 ssrc = 1;
required string kind = 2;
required string transport_id = 3;
required string codec_id = 4;
}
message ReceivedRtpStreamStats {
uint64 packets_received = 1;
int64 packets_lost = 2;
double jitter = 3;
required uint64 packets_received = 1;
required int64 packets_lost = 2;
required double jitter = 3;
}
message InboundRtpStreamStats {
string track_identifier = 1;
string mid = 2;
string remote_id = 3;
uint32 frames_decoded = 4;
uint32 key_frames_decoded = 5;
uint32 frames_rendered = 6;
uint32 frames_dropped = 7;
uint32 frame_width = 8;
uint32 frame_height = 9;
double frames_per_second = 10;
uint64 qp_sum = 11;
double total_decode_time = 12;
double total_inter_frame_delay = 13;
double total_squared_inter_frame_delay = 14;
uint32 pause_count = 15;
double total_pause_duration = 16;
uint32 freeze_count = 17;
double total_freeze_duration = 18;
double last_packet_received_timestamp = 19;
uint64 header_bytes_received = 20;
uint64 packets_discarded = 21;
uint64 fec_bytes_received = 22;
uint64 fec_packets_received = 23;
uint64 fec_packets_discarded = 24;
uint64 bytes_received = 25;
uint32 nack_count = 26;
uint32 fir_count = 27;
uint32 pli_count = 28;
double total_processing_delay = 29;
double estimated_playout_timestamp = 30;
double jitter_buffer_delay = 31;
double jitter_buffer_target_delay = 32;
uint64 jitter_buffer_emitted_count = 33;
double jitter_buffer_minimum_delay = 34;
uint64 total_samples_received = 35;
uint64 concealed_samples = 36;
uint64 silent_concealed_samples = 37;
uint64 concealment_events = 38;
uint64 inserted_samples_for_deceleration = 39;
uint64 removed_samples_for_acceleration = 40;
double audio_level = 41;
double total_audio_energy = 42;
double total_samples_duration = 43;
uint64 frames_received = 44;
string decoder_implementation = 45;
string playout_id = 46;
bool power_efficient_decoder = 47;
uint64 frames_assembled_from_multiple_packets = 48;
double total_assembly_time = 49;
uint64 retransmitted_packets_received = 50;
uint64 retransmitted_bytes_received = 51;
uint32 rtx_ssrc = 52;
uint32 fec_ssrc = 53;
required string track_identifier = 1;
required string mid = 2;
required string remote_id = 3;
required uint32 frames_decoded = 4;
required uint32 key_frames_decoded = 5;
required uint32 frames_rendered = 6;
required uint32 frames_dropped = 7;
required uint32 frame_width = 8;
required uint32 frame_height = 9;
required double frames_per_second = 10;
required uint64 qp_sum = 11;
required double total_decode_time = 12;
required double total_inter_frame_delay = 13;
required double total_squared_inter_frame_delay = 14;
required uint32 pause_count = 15;
required double total_pause_duration = 16;
required uint32 freeze_count = 17;
required double total_freeze_duration = 18;
required double last_packet_received_timestamp = 19;
required uint64 header_bytes_received = 20;
required uint64 packets_discarded = 21;
required uint64 fec_bytes_received = 22;
required uint64 fec_packets_received = 23;
required uint64 fec_packets_discarded = 24;
required uint64 bytes_received = 25;
required uint32 nack_count = 26;
required uint32 fir_count = 27;
required uint32 pli_count = 28;
required double total_processing_delay = 29;
required double estimated_playout_timestamp = 30;
required double jitter_buffer_delay = 31;
required double jitter_buffer_target_delay = 32;
required uint64 jitter_buffer_emitted_count = 33;
required double jitter_buffer_minimum_delay = 34;
required uint64 total_samples_received = 35;
required uint64 concealed_samples = 36;
required uint64 silent_concealed_samples = 37;
required uint64 concealment_events = 38;
required uint64 inserted_samples_for_deceleration = 39;
required uint64 removed_samples_for_acceleration = 40;
required double audio_level = 41;
required double total_audio_energy = 42;
required double total_samples_duration = 43;
required uint64 frames_received = 44;
required string decoder_implementation = 45;
required string playout_id = 46;
required bool power_efficient_decoder = 47;
required uint64 frames_assembled_from_multiple_packets = 48;
required double total_assembly_time = 49;
required uint64 retransmitted_packets_received = 50;
required uint64 retransmitted_bytes_received = 51;
required uint32 rtx_ssrc = 52;
required uint32 fec_ssrc = 53;
}
message SentRtpStreamStats {
uint64 packets_sent = 1;
uint64 bytes_sent = 2;
required uint64 packets_sent = 1;
required uint64 bytes_sent = 2;
}
message OutboundRtpStreamStats {
string mid = 1;
string media_source_id = 2;
string remote_id = 3;
string rid = 4;
uint64 header_bytes_sent = 5;
uint64 retransmitted_packets_sent = 6;
uint64 retransmitted_bytes_sent = 7;
uint32 rtx_ssrc = 8;
double target_bitrate = 9;
uint64 total_encoded_bytes_target = 10;
uint32 frame_width = 11;
uint32 frame_height = 12;
double frames_per_second = 13;
uint32 frames_sent = 14;
uint32 huge_frames_sent = 15;
uint32 frames_encoded = 16;
uint32 key_frames_encoded = 17;
uint64 qp_sum = 18;
double total_encode_time = 19;
double total_packet_send_delay = 20;
QualityLimitationReason quality_limitation_reason = 21;
required string mid = 1;
required string media_source_id = 2;
required string remote_id = 3;
required string rid = 4;
required uint64 header_bytes_sent = 5;
required uint64 retransmitted_packets_sent = 6;
required uint64 retransmitted_bytes_sent = 7;
required uint32 rtx_ssrc = 8;
required double target_bitrate = 9;
required uint64 total_encoded_bytes_target = 10;
required uint32 frame_width = 11;
required uint32 frame_height = 12;
required double frames_per_second = 13;
required uint32 frames_sent = 14;
required uint32 huge_frames_sent = 15;
required uint32 frames_encoded = 16;
required uint32 key_frames_encoded = 17;
required uint64 qp_sum = 18;
required double total_encode_time = 19;
required double total_packet_send_delay = 20;
required QualityLimitationReason quality_limitation_reason = 21;
map<string, double> quality_limitation_durations = 22;
uint32 quality_limitation_resolution_changes = 23;
uint32 nack_count = 24;
uint32 fir_count = 25;
uint32 pli_count = 26;
string encoder_implementation = 27;
bool power_efficient_encoder = 28;
bool active = 29;
string scalibility_mode = 30;
required uint32 quality_limitation_resolution_changes = 23;
required uint32 nack_count = 24;
required uint32 fir_count = 25;
required uint32 pli_count = 26;
required string encoder_implementation = 27;
required bool power_efficient_encoder = 28;
required bool active = 29;
required string scalability_mode = 30;
}
message RemoteInboundRtpStreamStats {
string local_id = 1;
double round_trip_time = 2;
double total_round_trip_time = 3;
double fraction_lost = 4;
uint64 round_trip_time_measurements = 5;
required string local_id = 1;
required double round_trip_time = 2;
required double total_round_trip_time = 3;
required double fraction_lost = 4;
required uint64 round_trip_time_measurements = 5;
}
message RemoteOutboundRtpStreamStats {
string local_id = 1;
double remote_timestamp = 2;
uint64 reports_sent = 3;
double round_trip_time = 4;
double total_round_trip_time = 5;
uint64 round_trip_time_measurements = 6;
required string local_id = 1;
required double remote_timestamp = 2;
required uint64 reports_sent = 3;
required double round_trip_time = 4;
required double total_round_trip_time = 5;
required uint64 round_trip_time_measurements = 6;
}
message MediaSourceStats {
string track_identifier = 1;
string kind = 2;
required string track_identifier = 1;
required string kind = 2;
}
message AudioSourceStats {
double audio_level = 1;
double total_audio_energy = 2;
double total_samples_duration = 3;
double echo_return_loss = 4;
double echo_return_loss_enhancement = 5;
double dropped_samples_duration = 6;
uint32 dropped_samples_events = 7;
double total_capture_delay = 8;
uint64 total_samples_captured = 9;
required double audio_level = 1;
required double total_audio_energy = 2;
required double total_samples_duration = 3;
required double echo_return_loss = 4;
required double echo_return_loss_enhancement = 5;
required double dropped_samples_duration = 6;
required uint32 dropped_samples_events = 7;
required double total_capture_delay = 8;
required uint64 total_samples_captured = 9;
}
message VideoSourceStats {
uint32 width = 1;
uint32 height = 2;
uint32 frames = 3;
double frames_per_second = 4;
required uint32 width = 1;
required uint32 height = 2;
required uint32 frames = 3;
required double frames_per_second = 4;
}
message AudioPlayoutStats {
string kind = 1;
double synthesized_samples_duration = 2;
uint32 synthesized_samples_events = 3;
double total_samples_duration = 4;
double total_playout_delay = 5;
uint64 total_samples_count = 6;
required string kind = 1;
required double synthesized_samples_duration = 2;
required uint32 synthesized_samples_events = 3;
required double total_samples_duration = 4;
required double total_playout_delay = 5;
required uint64 total_samples_count = 6;
}
message PeerConnectionStats {
uint32 data_channels_opened = 1;
uint32 data_channels_closed = 2;
required uint32 data_channels_opened = 1;
required uint32 data_channels_closed = 2;
}
message DataChannelStats {
string label = 1;
string protocol = 2;
int32 data_channel_identifier = 3;
required string label = 1;
required string protocol = 2;
required int32 data_channel_identifier = 3;
optional DataChannelState state = 4;
uint32 messages_sent = 5;
uint64 bytes_sent = 6;
uint32 messages_received = 7;
uint64 bytes_received = 8;
required uint32 messages_sent = 5;
required uint64 bytes_sent = 6;
required uint32 messages_received = 7;
required uint64 bytes_received = 8;
}
message TransportStats {
uint64 packets_sent = 1;
uint64 packets_received = 2;
uint64 bytes_sent = 3;
uint64 bytes_received = 4;
IceRole ice_role = 5;
string ice_local_username_fragment = 6;
required uint64 packets_sent = 1;
required uint64 packets_received = 2;
required uint64 bytes_sent = 3;
required uint64 bytes_received = 4;
required IceRole ice_role = 5;
required string ice_local_username_fragment = 6;
optional DtlsTransportState dtls_state = 7;
optional IceTransportState ice_state = 8;
string selected_candidate_pair_id = 9;
string local_certificate_id = 10;
string remote_certificate_id = 11;
string tls_version = 12;
string dtls_cipher = 13;
DtlsRole dtls_role = 14;
string srtp_cipher = 15;
uint32 selected_candidate_pair_changes = 16;
required string selected_candidate_pair_id = 9;
required string local_certificate_id = 10;
required string remote_certificate_id = 11;
required string tls_version = 12;
required string dtls_cipher = 13;
required DtlsRole dtls_role = 14;
required string srtp_cipher = 15;
required uint32 selected_candidate_pair_changes = 16;
}
message CandidatePairStats {
string transport_id = 1;
string local_candidate_id = 2;
string remote_candidate_id = 3;
required string transport_id = 1;
required string local_candidate_id = 2;
required string remote_candidate_id = 3;
optional IceCandidatePairState state = 4;
bool nominated = 5;
uint64 packets_sent = 6;
uint64 packets_received = 7;
uint64 bytes_sent = 8;
uint64 bytes_received = 9;
double last_packet_sent_timestamp = 10;
double last_packet_received_timestamp = 11;
double total_round_trip_time = 12;
double current_round_trip_time = 13;
double available_outgoing_bitrate = 14;
double available_incoming_bitrate = 15;
uint64 requests_received = 16;
uint64 requests_sent = 17;
uint64 responses_received = 18;
uint64 responses_sent = 19;
uint64 consent_requests_sent = 20;
uint32 packets_discarded_on_send = 21;
uint64 bytes_discarded_on_send = 22;
required bool nominated = 5;
required uint64 packets_sent = 6;
required uint64 packets_received = 7;
required uint64 bytes_sent = 8;
required uint64 bytes_received = 9;
required double last_packet_sent_timestamp = 10;
required double last_packet_received_timestamp = 11;
required double total_round_trip_time = 12;
required double current_round_trip_time = 13;
required double available_outgoing_bitrate = 14;
required double available_incoming_bitrate = 15;
required uint64 requests_received = 16;
required uint64 requests_sent = 17;
required uint64 responses_received = 18;
required uint64 responses_sent = 19;
required uint64 consent_requests_sent = 20;
required uint32 packets_discarded_on_send = 21;
required uint64 bytes_discarded_on_send = 22;
}
message IceCandidateStats {
string transport_id = 1;
string address = 2;
int32 port = 3;
string protocol = 4;
required string transport_id = 1;
required string address = 2;
required int32 port = 3;
required string protocol = 4;
optional IceCandidateType candidate_type = 5;
int32 priority = 6;
string url = 7;
required int32 priority = 6;
required string url = 7;
optional IceServerTransportProtocol relay_protocol = 8;
string foundation = 9;
string related_address = 10;
int32 related_port = 11;
string username_fragment = 12;
required string foundation = 9;
required string related_address = 10;
required int32 related_port = 11;
required string username_fragment = 12;
optional IceTcpCandidateType tcp_type = 13;
}
message CertificateStats {
string fingerprint = 1;
string fingerprint_algorithm = 2;
string base64_certificate = 3;
string issuer_certificate_id = 4;
required string fingerprint = 1;
required string fingerprint_algorithm = 2;
required string base64_certificate = 3;
required string issuer_certificate_id = 4;
}

82
src/protocols/track.proto
View File

@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
@@ -23,30 +23,30 @@ import "stats.proto";
// Create a new VideoTrack from a VideoSource
message CreateVideoTrackRequest {
string name = 1;
uint64 source_handle = 2;
required string name = 1;
required uint64 source_handle = 2;
}
message CreateVideoTrackResponse {
OwnedTrack track = 1;
required OwnedTrack track = 1;
}
// Create a new AudioTrack from a AudioSource
message CreateAudioTrackRequest {
string name = 1;
uint64 source_handle = 2;
required string name = 1;
required uint64 source_handle = 2;
}
message CreateAudioTrackResponse {
OwnedTrack track = 1;
required OwnedTrack track = 1;
}
message GetStatsRequest {
uint64 track_handle = 1;
required uint64 track_handle = 1;
}
message GetStatsResponse {
uint64 async_id = 1;
required uint64 async_id = 1;
}
message GetStatsCallback {
uint64 async_id = 1;
required uint64 async_id = 1;
optional string error = 2;
repeated RtcStats stats = 3;
}
@@ -78,34 +78,54 @@ enum StreamState {
}
message TrackPublicationInfo {
string sid = 1;
string name = 2;
TrackKind kind = 3;
TrackSource source = 4;
bool simulcasted = 5;
uint32 width = 6;
uint32 height = 7;
string mime_type = 8;
bool muted = 9;
bool remote = 10;
EncryptionType encryption_type = 11;
required string sid = 1;
required string name = 2;
required TrackKind kind = 3;
required TrackSource source = 4;
required bool simulcasted = 5;
required uint32 width = 6;
required uint32 height = 7;
required string mime_type = 8;
required bool muted = 9;
required bool remote = 10;
required EncryptionType encryption_type = 11;
}
message OwnedTrackPublication {
FfiOwnedHandle handle = 1;
TrackPublicationInfo info = 2;
required FfiOwnedHandle handle = 1;
required TrackPublicationInfo info = 2;
}
message TrackInfo {
string sid = 1;
string name = 2;
TrackKind kind = 3;
StreamState stream_state = 4;
bool muted = 5;
bool remote = 6;
required string sid = 1;
required string name = 2;
required TrackKind kind = 3;
required StreamState stream_state = 4;
required bool muted = 5;
required bool remote = 6;
}
message OwnedTrack {
FfiOwnedHandle handle = 1;
TrackInfo info = 2;
required FfiOwnedHandle handle = 1;
required TrackInfo info = 2;
}
// Mute/UnMute a track
message LocalTrackMuteRequest {
required uint64 track_handle = 1;
required bool mute = 2;
}
message LocalTrackMuteResponse {
required bool muted = 1;
}
// Enable/Disable a remote track
message EnableRemoteTrackRequest {
required uint64 track_handle = 1;
required bool enabled = 2;
}
message EnableRemoteTrackResponse {
required bool enabled = 1;
}

100
src/protocols/video_frame.proto
View File

@@ -12,52 +12,66 @@
// See the License for the specific language governing permissions and
// limitations under the License.
syntax = "proto3";
syntax = "proto2";
package livekit.proto;
option csharp_namespace = "LiveKit.Proto";
import "handle.proto";
import "track.proto";
// Create a new VideoStream
// VideoStream is used to receive video frames from a track
message NewVideoStreamRequest {
uint64 track_handle = 1;
VideoStreamType type = 2;
required uint64 track_handle = 1;
required VideoStreamType type = 2;
// Get the frame on a specific format
optional VideoBufferType format = 3;
bool normalize_stride = 4; // if true, stride will be set to width/chroma_width
optional bool normalize_stride = 4; // if true, stride will be set to width/chroma_width
}
message NewVideoStreamResponse { OwnedVideoStream stream = 1; }
message NewVideoStreamResponse { required OwnedVideoStream stream = 1; }
// Request a video stream from a participant
message VideoStreamFromParticipantRequest {
required uint64 participant_handle = 1;
required VideoStreamType type = 2;
required TrackSource track_source = 3;
optional VideoBufferType format = 4;
optional bool normalize_stride = 5;
}
message VideoStreamFromParticipantResponse { required OwnedVideoStream stream = 1;}
// Create a new VideoSource
// VideoSource is used to send video frame to a track
message NewVideoSourceRequest {
VideoSourceType type = 1;
required VideoSourceType type = 1;
// Used to determine which encodings to use + simulcast layers
// Most of the time it corresponds to the source resolution
VideoSourceResolution resolution = 2;
required VideoSourceResolution resolution = 2;
}
message NewVideoSourceResponse { OwnedVideoSource source = 1; }
message NewVideoSourceResponse { required OwnedVideoSource source = 1; }
// Push a frame to a VideoSource
message CaptureVideoFrameRequest {
uint64 source_handle = 1;
VideoBufferInfo buffer = 2;
int64 timestamp_us = 3; // In microseconds
VideoRotation rotation = 4;
required uint64 source_handle = 1;
required VideoBufferInfo buffer = 2;
required int64 timestamp_us = 3; // In microseconds
required VideoRotation rotation = 4;
}
message CaptureVideoFrameResponse {}
message VideoConvertRequest {
bool flip_y = 1;
VideoBufferInfo buffer = 2;
VideoBufferType dst_type = 3;
optional bool flip_y = 1;
required VideoBufferInfo buffer = 2;
required VideoBufferType dst_type = 3;
}
message VideoConvertResponse {
optional string error = 1;
OwnedVideoBuffer buffer = 2;
message VideoConvertResponse {
oneof message {
string error = 1;
OwnedVideoBuffer buffer = 2;
}
}
//
@@ -65,9 +79,9 @@ message VideoConvertResponse {
//
message VideoResolution {
uint32 width = 1;
uint32 height = 2;
double frame_rate = 3;
required uint32 width = 1;
required uint32 height = 2;
required double frame_rate = 3;
}
enum VideoCodec {
@@ -100,21 +114,21 @@ enum VideoBufferType {
message VideoBufferInfo {
message ComponentInfo {
uint64 data_ptr = 1;
uint32 stride = 2;
uint32 size = 3;
required uint64 data_ptr = 1;
required uint32 stride = 2;
required uint32 size = 3;
}
VideoBufferType type = 1;
uint32 width = 2;
uint32 height = 3;
uint64 data_ptr = 4;
uint32 stride = 6; // only for packed formats
required VideoBufferType type = 1;
required uint32 width = 2;
required uint32 height = 3;
required uint64 data_ptr = 4;
required uint32 stride = 6; // only for packed formats
repeated ComponentInfo components = 7;
}
message OwnedVideoBuffer {
FfiOwnedHandle handle = 1;
VideoBufferInfo info = 2;
required FfiOwnedHandle handle = 1;
required VideoBufferInfo info = 2;
}
//
@@ -128,16 +142,16 @@ enum VideoStreamType {
}
message VideoStreamInfo {
VideoStreamType type = 1;
required VideoStreamType type = 1;
}
message OwnedVideoStream {
FfiOwnedHandle handle = 1;
VideoStreamInfo info = 2;
required FfiOwnedHandle handle = 1;
required VideoStreamInfo info = 2;
}
message VideoStreamEvent {
uint64 stream_handle = 1;
required uint64 stream_handle = 1;
oneof message {
VideoFrameReceived frame_received = 2;
VideoStreamEOS eos = 3;
@@ -145,9 +159,9 @@ message VideoStreamEvent {
}
message VideoFrameReceived {
OwnedVideoBuffer buffer = 1;
int64 timestamp_us = 2; // In microseconds
VideoRotation rotation = 3;
required OwnedVideoBuffer buffer = 1;
required int64 timestamp_us = 2; // In microseconds
required VideoRotation rotation = 3;
}
message VideoStreamEOS {}
@@ -157,8 +171,8 @@ message VideoStreamEOS {}
//
message VideoSourceResolution {
uint32 width = 1;
uint32 height = 2;
required uint32 width = 1;
required uint32 height = 2;
}
enum VideoSourceType {
@@ -166,10 +180,10 @@ enum VideoSourceType {
}
message VideoSourceInfo {
VideoSourceType type = 1;
required VideoSourceType type = 1;
}
message OwnedVideoSource {
FfiOwnedHandle handle = 1;
VideoSourceInfo info = 2;
required FfiOwnedHandle handle = 1;
required VideoSourceInfo info = 2;
}

75
src/qml/CallPage.qml
View File

@@ -12,66 +12,46 @@ Kirigami.Page {
title: i18nc("@title", "Call")
VideoOutput {
id: video
RowLayout {
anchors.fill: parent
visible: false
Component.onCompleted: CallController.setVideoSink(video.videoSink)
}
VideoOutput {
id: viewFinder
anchors.centerIn: parent
VideoOutput {
id: viewFinder
ToolBar {
id: toolbar
Layout.fillWidth: parent.width / 2
Layout.preferredHeight: parent.height
anchors.horizontalCenter: parent.horizontalCenter
anchors.bottom: parent.bottom
anchors.bottomMargin: Kirigami.Units.gridUnit * 8
z: 1000
background: Kirigami.ShadowedRectangle {
color: Kirigami.Theme.backgroundColor
radius: 5
shadow {
size: 15
yOffset: 3
color: Qt.rgba(0, 0, 0, 0.2)
}
border {
color: Kirigami.ColorUtils.tintWithAlpha(Kirigami.Theme.backgroundColor, Kirigami.Theme.textColor, 0.2)
width: 1
}
Kirigami.Theme.inherit: false
Kirigami.Theme.colorSet: Kirigami.Theme.Window
Label {
text: "You"
}
RowLayout {
ToolButton {
id: cameraButton
icon.name: "camera-on-symbolic"
text: i18nc("@action:button", "Enable Camera")
display: AbstractButton.IconOnly
checkable: true
onClicked: CallController.toggleCamera()
}
ToolTip.text: text
ToolTip.visible: hovered
ToolTip.delay: Kirigami.Units.toolTipDelay
}
VideoOutput {
id: otherViewFinder
Layout.fillWidth: parent.width / 2
Layout.preferredHeight: parent.height
Label {
text: "Them"
}
}
}
LivekitVideoSink {
videoSink: viewFinder.videoSink
videoSink: otherViewFinder.videoSink
}
//Component.onCompleted: camera.start()
Component.onCompleted: camera.start()
Connections {
target: CallController
function onConnected(): void {
CallController.setCameraVideoSink(viewFinder.videoSink)
CallController.toggleCamera()
}
}
CaptureSession {
camera: Camera {
@@ -82,5 +62,4 @@ Kirigami.Page {
}
videoOutput: viewFinder
}
}