Skip to content

Commit

Permalink
Merge branch 'voip/sframe-support' into voip/livekit-group-call
Browse files Browse the repository at this point in the history
  • Loading branch information
cloudwebrtc committed Nov 23, 2023
2 parents cf6b603 + dc017b8 commit 246f81c
Show file tree
Hide file tree
Showing 5 changed files with 247 additions and 36 deletions.
114 changes: 83 additions & 31 deletions lib/src/voip/call.dart
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import 'package:webrtc_interface/webrtc_interface.dart';

import 'package:matrix/matrix.dart';
import 'package:matrix/src/utils/cached_stream_controller.dart';
import 'package:matrix/src/voip/sframe.dart';

/// https://github.com/matrix-org/matrix-doc/pull/2746
/// version 1
Expand Down Expand Up @@ -60,10 +61,13 @@ class WrappedMediaStream {

/// Current stream type, usermedia or screen-sharing
String purpose;
String? sframeKey;
bool audioMuted;
bool videoMuted;
final Client client;
VideoRenderer renderer;
List<RTCRtpSender> senders = [];
List<RTCRtpReceiver> receivers = [];
final bool isWeb;
final bool isGroupCall;
final RTCPeerConnection? pc;
Expand All @@ -77,18 +81,20 @@ class WrappedMediaStream {

void Function(MediaStream stream)? onNewStream;

WrappedMediaStream(
{this.stream,
this.pc,
required this.renderer,
required this.room,
required this.userId,
required this.purpose,
required this.client,
required this.audioMuted,
required this.videoMuted,
required this.isWeb,
required this.isGroupCall});
WrappedMediaStream({
this.stream,
this.pc,
required this.renderer,
required this.room,
required this.userId,
required this.purpose,
required this.client,
required this.audioMuted,
required this.videoMuted,
required this.isWeb,
required this.isGroupCall,
this.sframeKey,
});

/// Initialize the video renderer
Future<void> initialize() async {
Expand Down Expand Up @@ -247,6 +253,9 @@ class CallErrorCode {

/// We transferred the call off to somewhere else
static String Transfered = 'transferred';

/// Both parties need to enable sframe to establish a call
static String SFrameRequired = 'sframe_required';
}

class CallError extends Error {
Expand Down Expand Up @@ -303,6 +312,8 @@ class CallOptions {
late VoIP voip;
late Room room;
late List<Map<String, dynamic>> iceServers;
bool? sframe;
String? sframeKey;
}

/// A call session object
Expand All @@ -312,6 +323,7 @@ class CallSession {
CallType get type => opts.type;
Room get room => opts.room;
VoIP get voip => opts.voip;
bool get sframe => opts.sframe ?? false;
String? get groupCallId => opts.groupCallId;
String get callId => opts.callId;
String get localPartyId => opts.localPartyId;
Expand Down Expand Up @@ -539,11 +551,13 @@ class CallSession {
Future<void> sendAnswer(RTCSessionDescription answer) async {
final callCapabilities = CallCapabilities()
..dtmf = false
..transferee = false;
..transferee = false
..sframe = opts.sframe ?? false;

final metadata = SDPStreamMetadata({
localUserMediaStream!.stream!.id: SDPStreamPurpose(
purpose: SDPStreamMetadataPurpose.Usermedia,
sframeKey: opts.sframeKey,
audio_muted: localUserMediaStream!.stream!.getAudioTracks().isEmpty,
video_muted: localUserMediaStream!.stream!.getVideoTracks().isEmpty)
});
Expand Down Expand Up @@ -707,6 +721,7 @@ class CallSession {
wpstream
.setVideoMuted(metadata.sdpStreamMetadatas[streamId]!.video_muted);
wpstream.purpose = metadata.sdpStreamMetadatas[streamId]!.purpose;
wpstream.sframeKey = metadata.sdpStreamMetadatas[streamId]!.sframeKey;
} else {
Logs().i('Not found purpose for remote stream $streamId, remove it?');
wpstream.stopped = true;
Expand Down Expand Up @@ -821,14 +836,16 @@ class CallSession {
}
}

Future<void> addLocalStream(MediaStream stream, String purpose,
{bool addToPeerConnection = true}) async {
Future<void> addLocalStream(MediaStream stream, String purpose) async {
final existingStream =
getLocalStreams.where((element) => element.purpose == purpose);

WrappedMediaStream? newStream;
if (existingStream.isNotEmpty) {
existingStream.first.setNewStream(stream);
newStream = existingStream.first;
} else {
final newStream = WrappedMediaStream(
newStream = WrappedMediaStream(
renderer: voip.delegate.createRenderer(),
userId: client.userID!,
room: opts.room,
Expand All @@ -840,22 +857,31 @@ class CallSession {
isWeb: voip.delegate.isWeb,
isGroupCall: groupCallId != null,
pc: pc,
sframeKey: opts.sframeKey,
);
await newStream.initialize();
streams.add(newStream);
onStreamAdd.add(newStream);
}
onStreamAdd.add(newStream);

if (addToPeerConnection) {
if (purpose == SDPStreamMetadataPurpose.Screenshare) {
screensharingSenders.clear();
for (final track in stream.getTracks()) {
screensharingSenders.add(await pc!.addTrack(track, stream));
if (purpose == SDPStreamMetadataPurpose.Screenshare) {
screensharingSenders.clear();
for (final track in stream.getTracks()) {
final sender = await pc!.addTrack(track, stream);
newStream.senders.add(sender);
screensharingSenders.add(sender);
if (opts.sframe ?? false) {
await voip.handleAddRtpSender(callId, sender, newStream.sframeKey!);
}
} else if (purpose == SDPStreamMetadataPurpose.Usermedia) {
usermediaSenders.clear();
for (final track in stream.getTracks()) {
usermediaSenders.add(await pc!.addTrack(track, stream));
}
} else if (purpose == SDPStreamMetadataPurpose.Usermedia) {
usermediaSenders.clear();
for (final track in stream.getTracks()) {
final sender = await pc!.addTrack(track, stream);
newStream.senders.add(sender);
usermediaSenders.add(sender);
if (opts.sframe ?? false) {
await voip.handleAddRtpSender(callId, sender, newStream.sframeKey!);
}
}
}
Expand All @@ -871,7 +897,8 @@ class CallSession {
fireCallEvent(CallEvent.kFeedsChanged);
}

Future<void> _addRemoteStream(MediaStream stream) async {
Future<void> _addRemoteStream(
MediaStream stream, RTCRtpReceiver receiver) async {
//final userId = remoteUser.id;
final metadata = remoteSDPStreamMetadata!.sdpStreamMetadatas[stream.id];
if (metadata == null) {
Expand All @@ -883,13 +910,19 @@ class CallSession {
final purpose = metadata.purpose;
final audioMuted = metadata.audio_muted;
final videoMuted = metadata.video_muted;
final sFrameKey = metadata.sframeKey;

// Try to find a feed with the same purpose as the new stream,
// if we find it replace the old stream with the new one
final existingStream =
getRemoteStreams.where((element) => element.purpose == purpose);
if (existingStream.isNotEmpty) {
existingStream.first.setNewStream(stream);
existingStream.first.receivers.add(receiver);
if (opts.sframe ?? false) {
await voip.handleAddRtpReceiver(
callId, receiver, existingStream.first.sframeKey!);
}
} else {
final newStream = WrappedMediaStream(
renderer: voip.delegate.createRenderer(),
Expand All @@ -902,11 +935,16 @@ class CallSession {
videoMuted: videoMuted,
isWeb: voip.delegate.isWeb,
isGroupCall: groupCallId != null,
sframeKey: sFrameKey,
pc: pc,
);
await newStream.initialize();
streams.add(newStream);
newStream.receivers.add(receiver);
onStreamAdd.add(newStream);
if (opts.sframe ?? false) {
await voip.handleAddRtpReceiver(callId, receiver, newStream.sframeKey!);
}
}
fireCallEvent(CallEvent.kFeedsChanged);
Logs().i('Pushed remote stream (id="${stream.id}", purpose=$purpose)');
Expand Down Expand Up @@ -1030,7 +1068,13 @@ class CallSession {
} else {
// adding transceiver
Logs().d('[VOIP] adding track $newTrack to pc');
await pc!.addTrack(newTrack, localUserMediaStream!.stream!);
final sender =
await pc!.addTrack(newTrack, localUserMediaStream!.stream!);
localUserMediaStream!.senders.add(sender);
if (opts.sframe ?? false) {
await voip.handleAddRtpSender(
callId, sender, localUserMediaStream!.sframeKey!);
}
}
}
// for renderer to be able to show new video track
Expand Down Expand Up @@ -1099,17 +1143,20 @@ class CallSession {

final callCapabilities = CallCapabilities()
..dtmf = false
..sframe = opts.sframe ?? false
..transferee = false;

final metadata = SDPStreamMetadata({
if (localUserMediaStream != null)
localUserMediaStream!.stream!.id: SDPStreamPurpose(
purpose: SDPStreamMetadataPurpose.Usermedia,
sframeKey: opts.sframeKey,
audio_muted: localUserMediaStream!.audioMuted,
video_muted: localUserMediaStream!.videoMuted),
if (localScreenSharingStream != null)
localScreenSharingStream!.stream!.id: SDPStreamPurpose(
purpose: SDPStreamMetadataPurpose.Screenshare,
sframeKey: opts.sframeKey,
audio_muted: localScreenSharingStream!.audioMuted,
video_muted: localScreenSharingStream!.videoMuted),
});
Expand Down Expand Up @@ -1210,6 +1257,7 @@ class CallSession {
await cleanUp();
if (shouldEmit) {
onCallHangupNotifierForGroupCalls.add(this);
await voip.disposeSFrame(callId);
await voip.delegate.handleCallEnded(this);
fireCallEvent(CallEvent.kHangup);
if ((party == CallParty.kRemote && missedCall)) {
Expand Down Expand Up @@ -1262,7 +1310,9 @@ class CallSession {

final callCapabilities = CallCapabilities()
..dtmf = false
..transferee = false;
..transferee = false
..sframe = opts.sframe ?? false;

final metadata = _getLocalSDPStreamMetadata();
if (state == CallState.kCreateOffer) {
Logs().d('[glare] new invite sent about to be called');
Expand Down Expand Up @@ -1364,6 +1414,7 @@ class CallSession {
localCandidates.clear();
remoteCandidates.clear();
setCallState(CallState.kConnected);
await voip.setSframeEnabled(callId, opts.sframe ?? false);
// fix any state/race issues we had with sdp packets and cloned streams
await updateMuteStatus();
missedCall = false;
Expand Down Expand Up @@ -1431,7 +1482,8 @@ class CallSession {
sdpStreamMetadatas[wpstream.stream!.id] = SDPStreamPurpose(
purpose: wpstream.purpose,
audio_muted: wpstream.audioMuted,
video_muted: wpstream.videoMuted);
video_muted: wpstream.videoMuted,
sframeKey: opts.sframeKey);
}
}
final metadata = SDPStreamMetadata(sdpStreamMetadatas);
Expand Down Expand Up @@ -1494,7 +1546,7 @@ class CallSession {
pc.onTrack = (RTCTrackEvent event) async {
if (event.streams.isNotEmpty) {
final stream = event.streams[0];
await _addRemoteStream(stream);
await _addRemoteStream(stream, event.receiver!);
for (final track in stream.getTracks()) {
track.onEnded = () async {
if (stream.getTracks().isEmpty) {
Expand Down
102 changes: 102 additions & 0 deletions lib/src/voip/sframe.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
import 'dart:typed_data';

import 'package:webrtc_interface/webrtc_interface.dart';

import 'package:matrix/matrix.dart';

const String matrixWebRtcRatchetSalt = 'matrix-webrtc-ratchet-salt';

class FrameCryptorWrapper {
FrameCryptorWrapper(this.frameCryptor, this.keyProvider);
final FrameCryptor frameCryptor;
final KeyProvider keyProvider;
}

extension SframeExt on VoIP {
Future<KeyProvider> _initKeyProvider() async {
return await delegate.frameCryptorFactory!.createDefaultKeyProvider(
KeyProviderOptions(
sharedKey: false,
ratchetSalt: Uint8List.fromList(matrixWebRtcRatchetSalt.codeUnits),
ratchetWindowSize: 16,
),
);
}

Future<void> setSframeEnabled(String callId, bool enabled) async {
frameCryptors.forEach((key, fc) async {
if (key.startsWith(callId)) {
await fc.frameCryptor.setEnabled(enabled);
}
});
}

Future<void> disposeSFrame(String callId) async {
frameCryptors.removeWhere((key, fc) {
if (key.startsWith(callId)) {
fc.frameCryptor.dispose();
fc.keyProvider.dispose();
return true;
}
return false;
});
}

Future<void> handleAddRtpSender(
String callId,
RTCRtpSender sender,
String sFrameKey,
) async {
final trackId = sender.track?.id;
final kind = sender.track?.kind;
final id = '$callId-${kind!}_${trackId!}_sender';
if (!frameCryptors.containsKey(id)) {
final keyProvider = await _initKeyProvider();
final frameCryptor =
await delegate.frameCryptorFactory!.createFrameCryptorForRtpSender(
participantId: id,
sender: sender,
algorithm: Algorithm.kAesGcm,
keyProvider: keyProvider,
);
frameCryptor.onFrameCryptorStateChanged = (participantId, state) => Logs()
.d('[VoipPlugin] Encryptor onFrameCryptorStateChanged $participantId $state');
frameCryptors[id] = FrameCryptorWrapper(frameCryptor, keyProvider);
await keyProvider.setKey(
participantId: id,
index: 0,
key: Uint8List.fromList(sFrameKey.codeUnits),
);
await frameCryptor.setEnabled(true);
}
}

Future<void> handleAddRtpReceiver(
String callId,
RTCRtpReceiver receiver,
String sframeKey,
) async {
final trackId = receiver.track?.id;
final kind = receiver.track?.kind;
final id = '$callId-${kind!}_${trackId!}_receiver';
if (!frameCryptors.containsKey(id)) {
final keyProvider = await _initKeyProvider();
final frameCryptor =
await delegate.frameCryptorFactory!.createFrameCryptorForRtpReceiver(
participantId: id,
receiver: receiver,
algorithm: Algorithm.kAesGcm,
keyProvider: keyProvider,
);
frameCryptor.onFrameCryptorStateChanged = (participantId, state) => Logs()
.d('[VoipPlugin]Decryptor onFrameCryptorStateChanged $participantId $state');
frameCryptors[id] = FrameCryptorWrapper(frameCryptor, keyProvider);
await keyProvider.setKey(
participantId: id,
index: 0,
key: Uint8List.fromList(sframeKey.codeUnits),
);
await frameCryptor.setEnabled(true);
}
}
}
Loading

0 comments on commit 246f81c

Please sign in to comment.