import 'dart:async'; import 'dart:core'; import 'package:webrtc_interface/webrtc_interface.dart'; import 'package:sdp_transform/sdp_transform.dart' as sdp_transform; import '../matrix.dart'; /// Delegate WebRTC basic functionality. abstract class WebRTCDelegate { MediaDevices get mediaDevices; Future createPeerConnection( Map configuration, [Map constraints = const {}]); VideoRenderer createRenderer(); void playRingtone(); void stopRingtone(); void handleNewCall(CallSession session); void handleCallEnded(CallSession session); bool get isBackgroud; bool get isWeb; } /// The default life time for call events, in millisecond. const lifetimeMs = 10 * 1000; /// The length of time a call can be ringing for. const callTimeoutSec = 60; /// Wrapped MediaStream, used to adapt Widget to display class WrappedMediaStream { MediaStream? stream; final String userId; final Room room; /// Current stream type, usermedia or screen-sharing String purpose; bool audioMuted; bool videoMuted; final Client client; VideoRenderer renderer; final bool isWeb; /// for debug String get title => '$displayName:$purpose:a[$audioMuted]:v[$videoMuted]'; bool stopped = false; void Function(bool audioMuted, bool videoMuted)? onMuteStateChanged; void Function(MediaStream stream)? onNewStream; WrappedMediaStream( {this.stream, required this.renderer, required this.room, required this.userId, required this.purpose, required this.client, required this.audioMuted, required this.videoMuted, required this.isWeb}); /// Initialize the video renderer Future initialize() async { await renderer.initialize(); renderer.srcObject = stream; renderer.onResize = () { Logs().i( 'onResize [${stream!.id.substring(0, 8)}] ${renderer.videoWidth} x ${renderer.videoHeight}'); }; } Future dispose() async { renderer.srcObject = null; if (isLocal() && stream != null) { if (isWeb) { stream!.getTracks().forEach((element) { element.stop(); }); } await stream?.dispose(); stream = null; } } String get avatarName => getUser().calcDisplayname(mxidLocalPartFallback: false); String? get displayName => getUser().displayName; User getUser() { return room.getUserByMXIDSync(userId); } bool isLocal() { return userId == client.userID; } bool isAudioMuted() { return (stream != null && stream!.getAudioTracks().isEmpty) || audioMuted; } bool isVideoMuted() { return (stream != null && stream!.getVideoTracks().isEmpty) || videoMuted; } void setNewStream(MediaStream newStream) { stream = newStream; renderer.srcObject = stream; if (onNewStream != null) { onNewStream?.call(stream!); } } void setAudioMuted(bool muted) { audioMuted = muted; if (onMuteStateChanged != null) { onMuteStateChanged?.call(audioMuted, videoMuted); } } void setVideoMuted(bool muted) { videoMuted = muted; if (onMuteStateChanged != null) { onMuteStateChanged?.call(audioMuted, videoMuted); } } } // Call state enum CallState { /// The call is inilalized but not yet started kFledgling, /// The first time an invite is sent, the local has createdOffer kInviteSent, /// getUserMedia or getDisplayMedia has been called, /// but MediaStream has not yet been returned kWaitLocalMedia, /// The local has createdOffer kCreateOffer, /// Received a remote offer message and created a local Answer kCreateAnswer, /// Answer sdp is set, but ice is not connected kConnecting, /// WebRTC media stream is connected kConnected, /// The call was received, but no processing has been done yet. kRinging, /// End of call kEnded, } class CallErrorCode { /// The user chose to end the call static String UserHangup = 'user_hangup'; /// An error code when the local client failed to create an offer. static String LocalOfferFailed = 'local_offer_failed'; /// An error code when there is no local mic/camera to use. This may be because /// the hardware isn't plugged in, or the user has explicitly denied access. static String NoUserMedia = 'no_user_media'; /// Error code used when a call event failed to send /// because unknown devices were present in the room static String UnknownDevices = 'unknown_devices'; /// Error code used when we fail to send the invite /// for some reason other than there being unknown devices static String SendInvite = 'send_invite'; /// An answer could not be created static String CreateAnswer = 'create_answer'; /// Error code used when we fail to send the answer /// for some reason other than there being unknown devices static String SendAnswer = 'send_answer'; /// The session description from the other side could not be set static String SetRemoteDescription = 'set_remote_description'; /// The session description from this side could not be set static String SetLocalDescription = 'set_local_description'; /// A different device answered the call static String AnsweredElsewhere = 'answered_elsewhere'; /// No media connection could be established to the other party static String IceFailed = 'ice_failed'; /// The invite timed out whilst waiting for an answer static String InviteTimeout = 'invite_timeout'; /// The call was replaced by another call static String Replaced = 'replaced'; /// Signalling for the call could not be sent (other than the initial invite) static String SignallingFailed = 'signalling_timeout'; /// The remote party is busy static String UserBusy = 'user_busy'; /// We transferred the call off to somewhere else static String Transfered = 'transferred'; } class CallError extends Error { final String code; final String msg; final dynamic err; CallError(this.code, this.msg, this.err); @override String toString() { return '[$code] $msg, err: ${err.toString()}'; } } enum CallEvent { /// The call was hangup by the local|remote user. kHangup, /// The call state has changed kState, /// The call got some error. kError, /// Call transfer kReplaced, /// The value of isLocalOnHold() has changed kLocalHoldUnhold, /// The value of isRemoteOnHold() has changed kRemoteHoldUnhold, /// Feeds have changed kFeedsChanged, /// For sip calls. support in the future. kAssertedIdentityChanged, } enum CallType { kVoice, kVideo } enum CallDirection { kIncoming, kOutgoing } enum CallParty { kLocal, kRemote } /// Initialization parameters of the call session. class CallOptions { late String callId; late CallType type; late CallDirection dir; late String localPartyId; late VoIP voip; late Room room; late List> iceServers; } /// A call session object class CallSession { CallSession(this.opts); CallOptions opts; CallType get type => opts.type; Room get room => opts.room; VoIP get voip => opts.voip; String get callId => opts.callId; String get localPartyId => opts.localPartyId; String? get displayName => room.displayname; CallDirection get direction => opts.dir; CallState state = CallState.kFledgling; bool get isOutgoing => direction == CallDirection.kOutgoing; bool get isRinging => state == CallState.kRinging; RTCPeerConnection? pc; List remoteCandidates = []; List localCandidates = []; late AssertedIdentity remoteAssertedIdentity; bool get callHasEnded => state == CallState.kEnded; bool iceGatheringFinished = false; bool inviteOrAnswerSent = false; bool localHold = false; bool remoteOnHold = false; bool _answeredByUs = false; bool speakerOn = false; bool makingOffer = false; bool ignoreOffer = false; String facingMode = 'user'; Client get client => opts.room.client; String? remotePartyId; late User remoteUser; late CallParty hangupParty; late String hangupReason; late CallError lastError; SDPStreamMetadata? remoteSDPStreamMetadata; List usermediaSenders = []; List screensharingSenders = []; List streams = []; List get getLocalStreams => streams.where((element) => element.isLocal()).toList(); List get getRemoteStreams => streams.where((element) => !element.isLocal()).toList(); WrappedMediaStream? get localUserMediaStream { final stream = getLocalStreams.where( (element) => element.purpose == SDPStreamMetadataPurpose.Usermedia); if (stream.isNotEmpty) { return stream.first; } return null; } WrappedMediaStream? get localScreenSharingStream { final stream = getLocalStreams.where( (element) => element.purpose == SDPStreamMetadataPurpose.Screenshare); if (stream.isNotEmpty) { return stream.first; } return null; } WrappedMediaStream? get remoteUserMediaStream { final stream = getRemoteStreams.where( (element) => element.purpose == SDPStreamMetadataPurpose.Usermedia); if (stream.isNotEmpty) { return stream.first; } return null; } WrappedMediaStream? get remoteScreenSharingStream { final stream = getRemoteStreams.where( (element) => element.purpose == SDPStreamMetadataPurpose.Screenshare); if (stream.isNotEmpty) { return stream.first; } return null; } final _callStateController = StreamController.broadcast(sync: true); Stream get onCallStateChanged => _callStateController.stream; final _callEventController = StreamController.broadcast(sync: true); Stream get onCallEventChanged => _callEventController.stream; Timer? inviteTimer; Timer? ringingTimer; Future initOutboundCall(CallType type) async { await _preparePeerConnection(); setCallState(CallState.kCreateOffer); final stream = await _getUserMedia(type); if (stream != null) { _addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); } } Future initWithInvite(CallType type, RTCSessionDescription offer, SDPStreamMetadata? metadata, int lifetime) async { await _preparePeerConnection(); final stream = await _getUserMedia(type); if (stream != null) { _addLocalStream(stream, SDPStreamMetadataPurpose.Usermedia); } if (metadata != null) { _updateRemoteSDPStreamMetadata(metadata); } await pc!.setRemoteDescription(offer); setCallState(CallState.kRinging); ringingTimer = Timer(Duration(milliseconds: 30000 - lifetime), () { if (state == CallState.kRinging) { Logs().v('[VOIP] Call invite has expired. Hanging up.'); hangupParty = CallParty.kRemote; // effectively fireCallEvent(CallEvent.kHangup); hangup(CallErrorCode.InviteTimeout); } ringingTimer?.cancel(); ringingTimer = null; }); } void initWithHangup() { setCallState(CallState.kEnded); } void onAnswerReceived( RTCSessionDescription answer, SDPStreamMetadata? metadata) async { if (metadata != null) { _updateRemoteSDPStreamMetadata(metadata); } if (direction == CallDirection.kOutgoing) { setCallState(CallState.kConnecting); await pc!.setRemoteDescription(answer); remoteCandidates.forEach((candidate) => pc!.addCandidate(candidate)); } } void onNegotiateReceived( SDPStreamMetadata? metadata, RTCSessionDescription description) async { final polite = direction == CallDirection.kIncoming; // Here we follow the perfect negotiation logic from // https://developer.mozilla.org/en-US/docs/Web/API/WebRTC_API/Perfect_negotiation final offerCollision = ((description.type == 'offer') && (makingOffer || pc!.signalingState != RTCSignalingState.RTCSignalingStateStable)); ignoreOffer = !polite && offerCollision; if (ignoreOffer) { Logs().i('Ignoring colliding negotiate event because we\'re impolite'); return; } final prevLocalOnHold = await isLocalOnHold(); if (metadata != null) { _updateRemoteSDPStreamMetadata(metadata); } try { await pc!.setRemoteDescription(description); if (description.type == 'offer') { final answer = await pc!.createAnswer({}); await room.sendCallNegotiate( callId, lifetimeMs, localPartyId, answer.sdp!, type: answer.type!); await pc!.setLocalDescription(answer); } } catch (e) { _getLocalOfferFailed(e); Logs().e('[VOIP] onNegotiateReceived => ${e.toString()}'); return; } final newLocalOnHold = await isLocalOnHold(); if (prevLocalOnHold != newLocalOnHold) { localHold = newLocalOnHold; fireCallEvent(CallEvent.kLocalHoldUnhold); } } void _updateRemoteSDPStreamMetadata(SDPStreamMetadata metadata) { remoteSDPStreamMetadata = metadata; remoteSDPStreamMetadata!.sdpStreamMetadatas .forEach((streamId, sdpStreamMetadata) { Logs().i( 'Stream purpose update: \nid = "$streamId", \npurpose = "${sdpStreamMetadata.purpose}", \naudio_muted = ${sdpStreamMetadata.audio_muted}, \nvideo_muted = ${sdpStreamMetadata.video_muted}'); }); getRemoteStreams.forEach((wpstream) { final streamId = wpstream.stream!.id; final purpose = metadata.sdpStreamMetadatas[streamId]; if (purpose != null) { wpstream .setAudioMuted(metadata.sdpStreamMetadatas[streamId]!.audio_muted); wpstream .setVideoMuted(metadata.sdpStreamMetadatas[streamId]!.video_muted); wpstream.purpose = metadata.sdpStreamMetadatas[streamId]!.purpose; } else { Logs().i('Not found purpose for remote stream $streamId, remove it?'); wpstream.stopped = true; fireCallEvent(CallEvent.kFeedsChanged); } }); } void onSDPStreamMetadataReceived(SDPStreamMetadata metadata) async { _updateRemoteSDPStreamMetadata(metadata); fireCallEvent(CallEvent.kFeedsChanged); } void onCandidatesReceived(List candidates) { candidates.forEach((json) async { final candidate = RTCIceCandidate( json['candidate'], json['sdpMid'] ?? '', json['sdpMLineIndex']?.round() ?? 0, ); if (pc != null && inviteOrAnswerSent && remotePartyId != null) { try { await pc!.addCandidate(candidate); } catch (e) { Logs().e('[VOIP] onCandidatesReceived => ${e.toString()}'); } } else { remoteCandidates.add(candidate); } }); if (pc != null && pc!.iceConnectionState == RTCIceConnectionState.RTCIceConnectionStateDisconnected) { restartIce(); } } void onAssertedIdentityReceived(AssertedIdentity identity) async { remoteAssertedIdentity = identity; fireCallEvent(CallEvent.kAssertedIdentityChanged); } bool get screensharingEnabled => localScreenSharingStream != null; Future setScreensharingEnabled(bool enabled) async { // Skip if there is nothing to do if (enabled && localScreenSharingStream != null) { Logs().w( 'There is already a screensharing stream - there is nothing to do!'); return true; } else if (!enabled && localScreenSharingStream == null) { Logs().w( 'There already isn\'t a screensharing stream - there is nothing to do!'); return false; } Logs().d('Set screensharing enabled? $enabled'); if (enabled) { try { final stream = await _getDisplayMedia(); if (stream == null) { return false; } stream.getVideoTracks().forEach((track) { track.onEnded = () { setScreensharingEnabled(false); }; }); _addLocalStream(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { fireCallEvent(CallEvent.kError); lastError = CallError(CallErrorCode.NoUserMedia, 'Failed to get screen-sharing stream: ', err); return false; } } else { for (final sender in screensharingSenders) { await pc!.removeTrack(sender); } for (final track in localScreenSharingStream!.stream!.getTracks()) { await track.stop(); } localScreenSharingStream!.stopped = true; await _removeStream(localScreenSharingStream!.stream!); fireCallEvent(CallEvent.kFeedsChanged); return false; } } void _addLocalStream(MediaStream stream, String purpose, {bool addToPeerConnection = true}) async { final existingStream = getLocalStreams.where((element) => element.purpose == purpose); if (existingStream.isNotEmpty) { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( renderer: voip.delegate.createRenderer(), userId: client.userID!, room: opts.room, stream: stream, purpose: purpose, client: client, audioMuted: stream.getAudioTracks().isEmpty, videoMuted: stream.getVideoTracks().isEmpty, isWeb: voip.delegate.isWeb, ); await newStream.initialize(); streams.add(newStream); fireCallEvent(CallEvent.kFeedsChanged); } if (addToPeerConnection) { if (purpose == SDPStreamMetadataPurpose.Screenshare) { screensharingSenders.clear(); stream.getTracks().forEach((track) async { screensharingSenders.add(await pc!.addTrack(track, stream)); }); } else if (purpose == SDPStreamMetadataPurpose.Usermedia) { usermediaSenders.clear(); stream.getTracks().forEach((track) async { usermediaSenders.add(await pc!.addTrack(track, stream)); }); } fireCallEvent(CallEvent.kFeedsChanged); } if (purpose == SDPStreamMetadataPurpose.Usermedia) { speakerOn = type == CallType.kVideo; if (!voip.delegate.isWeb && !voip.delegate.isBackgroud) { final audioTrack = stream.getAudioTracks()[0]; audioTrack.enableSpeakerphone(speakerOn); } } } void _addRemoteStream(MediaStream stream) async { //const userId = this.getOpponentMember().userId; final metadata = remoteSDPStreamMetadata!.sdpStreamMetadatas[stream.id]; if (metadata == null) { Logs().i( 'Ignoring stream with id ${stream.id} because we didn\'t get any metadata about it'); return; } final purpose = metadata.purpose; final audioMuted = metadata.audio_muted; final videoMuted = metadata.video_muted; // Try to find a feed with the same purpose as the new stream, // if we find it replace the old stream with the new one final existingStream = getRemoteStreams.where((element) => element.purpose == purpose); if (existingStream.isNotEmpty) { existingStream.first.setNewStream(stream); } else { final newStream = WrappedMediaStream( renderer: voip.delegate.createRenderer(), userId: remoteUser.id, room: opts.room, stream: stream, purpose: purpose, client: client, audioMuted: audioMuted, videoMuted: videoMuted, isWeb: voip.delegate.isWeb, ); await newStream.initialize(); streams.add(newStream); } fireCallEvent(CallEvent.kFeedsChanged); Logs().i('Pushed remote stream (id="${stream.id}", purpose=$purpose)'); } void setCallState(CallState newState) { state = newState; _callStateController.add(newState); fireCallEvent(CallEvent.kState); } void setLocalVideoMuted(bool muted) { localUserMediaStream?.setVideoMuted(muted); _updateMuteStatus(); } bool get isLocalVideoMuted => localUserMediaStream?.isVideoMuted() ?? false; void setMicrophoneMuted(bool muted) { localUserMediaStream?.setAudioMuted(muted); _updateMuteStatus(); } bool get isMicrophoneMuted => localUserMediaStream?.isAudioMuted() ?? false; void setRemoteOnHold(bool onHold) async { if (isRemoteOnHold == onHold) return; remoteOnHold = onHold; final transceivers = await pc!.getTransceivers(); for (final transceiver in transceivers) { await transceiver.setDirection(onHold ? TransceiverDirection.SendOnly : TransceiverDirection.SendRecv); } _updateMuteStatus(); fireCallEvent(CallEvent.kRemoteHoldUnhold); } bool get isRemoteOnHold => remoteOnHold; Future isLocalOnHold() async { if (state != CallState.kConnected) return false; var callOnHold = true; // We consider a call to be on hold only if *all* the tracks are on hold // (is this the right thing to do?) final transceivers = await pc!.getTransceivers(); for (final transceiver in transceivers) { final currentDirection = await transceiver.getCurrentDirection(); Logs() .i('transceiver.currentDirection = ${currentDirection?.toString()}'); final trackOnHold = (currentDirection == TransceiverDirection.Inactive || currentDirection == TransceiverDirection.RecvOnly); if (!trackOnHold) { callOnHold = false; } } return callOnHold; } void answer() async { if (inviteOrAnswerSent) { return; } // stop play ringtone voip.delegate.stopRingtone(); if (direction == CallDirection.kIncoming) { setCallState(CallState.kCreateAnswer); final answer = await pc!.createAnswer({}); remoteCandidates.forEach((candidate) => pc!.addCandidate(candidate)); final callCapabilities = CallCapabilities() ..dtmf = false ..transferee = false; final metadata = SDPStreamMetadata({ localUserMediaStream!.stream!.id: SDPStreamPurpose( purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: localUserMediaStream!.stream!.getAudioTracks().isEmpty, video_muted: localUserMediaStream!.stream!.getVideoTracks().isEmpty) }); final res = await room.answerCall(callId, answer.sdp!, localPartyId, type: answer.type!, capabilities: callCapabilities, metadata: metadata); Logs().v('[VOIP] answer res => $res'); await pc!.setLocalDescription(answer); setCallState(CallState.kConnecting); inviteOrAnswerSent = true; _answeredByUs = true; } } /// Reject a call /// This used to be done by calling hangup, but is a separate method and protocol /// event as of MSC2746. /// void reject() { if (state != CallState.kRinging) { Logs().e('[VOIP] Call must be in \'ringing\' state to reject!'); return; } Logs().d('[VOIP] Rejecting call: $callId'); terminate(CallParty.kLocal, CallErrorCode.UserHangup, true); room.sendCallReject(callId, lifetimeMs, localPartyId); } void hangup([String? reason, bool suppressEvent = true]) async { // stop play ringtone voip.delegate.stopRingtone(); terminate( CallParty.kLocal, reason ?? CallErrorCode.UserHangup, !suppressEvent); try { final res = await room.hangupCall(callId, localPartyId, 'userHangup'); Logs().v('[VOIP] hangup res => $res'); } catch (e) { Logs().v('[VOIP] hangup error => ${e.toString()}'); } } void sendDTMF(String tones) async { final senders = await pc!.getSenders(); for (final sender in senders) { if (sender.track != null && sender.track!.kind == 'audio') { await sender.dtmfSender.insertDTMF(tones); return; } } Logs().e('Unable to find a track to send DTMF on'); } void terminate(CallParty party, String hangupReason, bool shouldEmit) async { if (state == CallState.kEnded) { return; } inviteTimer?.cancel(); inviteTimer = null; ringingTimer?.cancel(); ringingTimer = null; hangupParty = party; hangupReason = hangupReason; setCallState(CallState.kEnded); voip.currentCID = null; voip.calls.remove(callId); cleanUp(); voip.delegate.handleCallEnded(this); if (shouldEmit) { fireCallEvent(CallEvent.kHangup); } } void onRejectReceived(String? reason) { Logs().v('[VOIP] Reject received for call ID ' + callId); // No need to check party_id for reject because if we'd received either // an answer or reject, we wouldn't be in state InviteSent final shouldTerminate = (state == CallState.kFledgling && direction == CallDirection.kIncoming) || CallState.kInviteSent == state || CallState.kRinging == state; if (shouldTerminate) { terminate(CallParty.kRemote, reason ?? CallErrorCode.UserHangup, true); } else { Logs().e('Call is in state: ${state.toString()}: ignoring reject'); } } Future _gotLocalOffer(RTCSessionDescription offer) async { if (callHasEnded) { Logs().d( 'Ignoring newly created offer on call ID ${opts.callId} because the call has ended'); return; } try { await pc!.setLocalDescription(offer); } catch (err) { Logs().d('Error setting local description! ${err.toString()}'); terminate(CallParty.kLocal, CallErrorCode.SetLocalDescription, true); return; } if (callHasEnded) return; final callCapabilities = CallCapabilities() ..dtmf = false ..transferee = false; final metadata = _getLocalSDPStreamMetadata(); if (state == CallState.kCreateOffer) { await room.inviteToCall( callId, lifetimeMs, localPartyId, null, offer.sdp!, capabilities: callCapabilities, metadata: metadata); inviteOrAnswerSent = true; setCallState(CallState.kInviteSent); inviteTimer = Timer(Duration(seconds: callTimeoutSec), () { if (state == CallState.kInviteSent) { hangup(CallErrorCode.InviteTimeout, false); } inviteTimer?.cancel(); inviteTimer = null; }); } else { await room.sendCallNegotiate(callId, lifetimeMs, localPartyId, offer.sdp!, type: offer.type!, capabilities: callCapabilities, metadata: metadata); } } void onNegotiationNeeded() async { Logs().i('Negotiation is needed!'); makingOffer = true; try { final offer = await pc!.createOffer({}); await _gotLocalOffer(offer); } catch (e) { _getLocalOfferFailed(e); return; } finally { makingOffer = false; } } Future _preparePeerConnection() async { try { pc = await _createPeerConnection(); pc!.onRenegotiationNeeded = onNegotiationNeeded; pc!.onIceCandidate = (RTCIceCandidate candidate) async { //Logs().v('[VOIP] onIceCandidate => ${candidate.toMap().toString()}'); localCandidates.add(candidate); }; pc!.onIceGatheringState = (RTCIceGatheringState state) async { Logs().v('[VOIP] IceGatheringState => ${state.toString()}'); if (state == RTCIceGatheringState.RTCIceGatheringStateGathering) { Timer(Duration(seconds: 3), () async { if (!iceGatheringFinished) { iceGatheringFinished = true; await _candidateReady(); } }); } if (state == RTCIceGatheringState.RTCIceGatheringStateComplete) { if (!iceGatheringFinished) { iceGatheringFinished = true; await _candidateReady(); } } }; pc!.onIceConnectionState = (RTCIceConnectionState state) { Logs().v('[VOIP] RTCIceConnectionState => ${state.toString()}'); if (state == RTCIceConnectionState.RTCIceConnectionStateConnected) { localCandidates.clear(); remoteCandidates.clear(); setCallState(CallState.kConnected); } else if (state == RTCIceConnectionState.RTCIceConnectionStateFailed) { hangup(CallErrorCode.IceFailed, false); } }; } catch (e) { Logs().v('[VOIP] prepareMediaStream error => ${e.toString()}'); } } void onAnsweredElsewhere(String msg) { Logs().d('Call ID $callId answered elsewhere'); terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); } void cleanUp() async { streams.forEach((stream) { stream.dispose(); }); streams.clear(); if (pc != null) { await pc!.close(); await pc!.dispose(); } } void _updateMuteStatus() async { final micShouldBeMuted = (localUserMediaStream != null && localUserMediaStream!.isAudioMuted()) || remoteOnHold; final vidShouldBeMuted = (localUserMediaStream != null && localUserMediaStream!.isVideoMuted()) || remoteOnHold; _setTracksEnabled(localUserMediaStream?.stream!.getAudioTracks() ?? [], !micShouldBeMuted); _setTracksEnabled(localUserMediaStream?.stream!.getVideoTracks() ?? [], !vidShouldBeMuted); await opts.room.sendSDPStreamMetadataChanged( callId, localPartyId, _getLocalSDPStreamMetadata()); } void _setTracksEnabled(List tracks, bool enabled) { tracks.forEach((track) async { track.enabled = enabled; }); } SDPStreamMetadata _getLocalSDPStreamMetadata() { final sdpStreamMetadatas = {}; for (final wpstream in getLocalStreams) { sdpStreamMetadatas[wpstream.stream!.id] = SDPStreamPurpose( purpose: wpstream.purpose, audio_muted: wpstream.audioMuted, video_muted: wpstream.videoMuted); } final metadata = SDPStreamMetadata(sdpStreamMetadatas); Logs().v('Got local SDPStreamMetadata ${metadata.toJson().toString()}'); return metadata; } void restartIce() async { Logs().v('[VOIP] iceRestart.'); // Needs restart ice on session.pc and renegotiation. iceGatheringFinished = false; final desc = await pc!.createOffer(_getOfferAnswerConstraints(iceRestart: true)); await pc!.setLocalDescription(desc); localCandidates.clear(); } Future _getUserMedia(CallType type) async { final mediaConstraints = { 'audio': true, 'video': type == CallType.kVideo ? { 'mandatory': { 'minWidth': '640', 'minHeight': '480', 'minFrameRate': '30', }, 'facingMode': 'user', 'optional': [], } : false, }; try { return await voip.delegate.mediaDevices.getUserMedia(mediaConstraints); } catch (e) { _getUserMediaFailed(e); } return null; } Future _getDisplayMedia() async { final mediaConstraints = { 'audio': false, 'video': true, }; try { return await voip.delegate.mediaDevices.getDisplayMedia(mediaConstraints); } catch (e) { _getUserMediaFailed(e); } return null; } Future _createPeerConnection() async { final configuration = { 'iceServers': opts.iceServers, 'sdpSemantics': 'unified-plan' }; final pc = await voip.delegate.createPeerConnection(configuration); pc.onTrack = (RTCTrackEvent event) { if (event.streams.isNotEmpty) { final stream = event.streams[0]; _addRemoteStream(stream); } }; return pc; } void tryRemoveStopedStreams() { final removedStreams = {}; streams.forEach((stream) { if (stream.stopped) { removedStreams[stream.stream!.id] = stream; } }); streams .removeWhere((stream) => removedStreams.containsKey(stream.stream!.id)); removedStreams.forEach((id, element) { _removeStream(element.stream!); }); } Future _removeStream(MediaStream stream) async { Logs().v('Removing feed with stream id ${stream.id}'); final it = streams.where((element) => element.stream!.id == stream.id); if (it.isEmpty) { Logs().v('Didn\'t find the feed with stream id ${stream.id} to delete'); return; } final wpstream = it.first; streams.removeWhere((element) => element.stream!.id == stream.id); fireCallEvent(CallEvent.kFeedsChanged); await wpstream.dispose(); } Map _getOfferAnswerConstraints({bool iceRestart = false}) { return { 'mandatory': {if (iceRestart) 'IceRestart': true}, 'optional': [], }; } Future _candidateReady() async { /* Currently, trickle-ice is not supported, so it will take a long time to wait to collect all the canidates, set the timeout for collection canidates to speed up the connection. */ try { final candidates = >[]; localCandidates.forEach((element) { candidates.add(element.toMap()); }); final res = await room.sendCallCandidates(callId, localPartyId, candidates); Logs().v('[VOIP] sendCallCandidates res => $res'); } catch (e) { Logs().v('[VOIP] sendCallCandidates e => ${e.toString()}'); } } void fireCallEvent(CallEvent event) { _callEventController.add(event); Logs().i('CallEvent: ${event.toString()}'); switch (event) { case CallEvent.kFeedsChanged: break; case CallEvent.kState: Logs().i('CallState: ${state.toString()}'); break; case CallEvent.kError: break; case CallEvent.kHangup: break; case CallEvent.kReplaced: break; case CallEvent.kLocalHoldUnhold: break; case CallEvent.kRemoteHoldUnhold: break; case CallEvent.kAssertedIdentityChanged: break; } } void _getLocalOfferFailed(dynamic err) { Logs().e('Failed to get local offer ${err.toString()}'); fireCallEvent(CallEvent.kError); lastError = CallError( CallErrorCode.LocalOfferFailed, 'Failed to get local offer!', err); terminate(CallParty.kLocal, CallErrorCode.LocalOfferFailed, false); } void _getUserMediaFailed(dynamic err) { if (state != CallState.kConnected) { Logs().w('Failed to get user media - ending call ${err.toString()}'); fireCallEvent(CallEvent.kError); lastError = CallError( CallErrorCode.NoUserMedia, 'Couldn\'t start capturing media! Is your microphone set up and does this app have permission?', err); terminate(CallParty.kLocal, CallErrorCode.NoUserMedia, false); } } void onSelectAnswerReceived(String? selectedPartyId) { if (direction != CallDirection.kIncoming) { Logs().w('Got select_answer for an outbound call: ignoring'); return; } if (selectedPartyId == null) { Logs().w( 'Got nonsensical select_answer with null/undefined selected_party_id: ignoring'); return; } if (selectedPartyId != localPartyId) { Logs().w( 'Got select_answer for party ID $selectedPartyId: we are party ID $localPartyId.'); // The other party has picked somebody else's answer terminate(CallParty.kRemote, CallErrorCode.AnsweredElsewhere, true); } } } class VoIP { TurnServerCredentials? _turnServerCredentials; Map calls = {}; String? currentCID; String? get localPartyId => client.deviceID; final Client client; final WebRTCDelegate delegate; VoIP(this.client, this.delegate) : super() { client.onCallInvite.stream.listen(onCallInvite); client.onCallAnswer.stream.listen(onCallAnswer); client.onCallCandidates.stream.listen(onCallCandidates); client.onCallHangup.stream.listen(onCallHangup); client.onCallReject.stream.listen(onCallReject); client.onCallNegotiate.stream.listen(onCallNegotiate); client.onCallReplaces.stream.listen(onCallReplaces); client.onCallSelectAnswer.stream.listen(onCallSelectAnswer); client.onSDPStreamMetadataChangedReceived.stream .listen(onSDPStreamMetadataChangedReceived); client.onAssertedIdentityReceived.stream.listen(onAssertedIdentityReceived); } Future onCallInvite(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } Logs().v( '[VOIP] onCallInvite ${event.senderId} => ${client.userID}, \ncontent => ${event.content.toString()}'); final String callId = event.content['call_id']; final String partyId = event.content['party_id']; final int lifetime = event.content['lifetime']; if (currentCID != null) { // Only one session at a time. Logs().v('[VOIP] onCallInvite: There is already a session.'); await event.room.hangupCall(callId, localPartyId!, 'userBusy'); return; } if (calls[callId] != null) { // Session already exist. Logs().v('[VOIP] onCallInvite: Session [$callId] already exist.'); return; } if (event.content['capabilities'] != null) { final capabilities = CallCapabilities.fromJson(event.content['capabilities']); Logs().v( '[VOIP] CallCapabilities: dtmf => ${capabilities.dtmf}, transferee => ${capabilities.transferee}'); } var callType = CallType.kVoice; SDPStreamMetadata? sdpStreamMetadata; if (event.content[sdpStreamMetadataKey] != null) { sdpStreamMetadata = SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); sdpStreamMetadata.sdpStreamMetadatas .forEach((streamId, SDPStreamPurpose purpose) { Logs().v( '[VOIP] [$streamId] => purpose: ${purpose.purpose}, audioMuted: ${purpose.audio_muted}, videoMuted: ${purpose.video_muted}'); if (!purpose.video_muted) { callType = CallType.kVideo; } }); } else { callType = getCallType(event.content['offer']['sdp']); } final opts = CallOptions() ..voip = this ..callId = callId ..dir = CallDirection.kIncoming ..type = callType ..room = event.room ..localPartyId = localPartyId! ..iceServers = await getIceSevers(); final newCall = createNewCall(opts); newCall.remotePartyId = partyId; newCall.remoteUser = event.sender; final offer = RTCSessionDescription( event.content['offer']['sdp'], event.content['offer']['type'], ); await newCall .initWithInvite(callType, offer, sdpStreamMetadata, lifetime) .then((_) { // Popup CallingPage for incoming call. if (!delegate.isBackgroud) { delegate.handleNewCall(newCall); } }); currentCID = callId; if (delegate.isBackgroud) { /// Forced to enable signaling synchronization until the end of the call. client.backgroundSync = true; ///TODO: notify the callkeep that the call is incoming. } // Play ringtone delegate.playRingtone(); } void onCallAnswer(Event event) async { Logs().v('[VOIP] onCallAnswer => ${event.content.toString()}'); final String callId = event.content['call_id']; final String partyId = event.content['party_id']; final call = calls[callId]; if (call != null) { if (event.senderId == client.userID) { // Ignore messages to yourself. if (!call._answeredByUs) { delegate.stopRingtone(); } if (call.state == CallState.kRinging) { call.onAnsweredElsewhere('Call ID ' + callId + ' answered elsewhere'); } return; } call.remotePartyId = partyId; call.remoteUser = event.sender; final answer = RTCSessionDescription( event.content['answer']['sdp'], event.content['answer']['type']); SDPStreamMetadata? metadata; if (event.content[sdpStreamMetadataKey] != null) { metadata = SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); } call.onAnswerReceived(answer, metadata); /// Send select_answer event. await event.room.selectCallAnswer( callId, lifetimeMs, localPartyId!, call.remotePartyId!); } else { Logs().v('[VOIP] onCallAnswer: Session [$callId] not found!'); } } void onCallCandidates(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } Logs().v('[VOIP] onCallCandidates => ${event.content.toString()}'); final String callId = event.content['call_id']; final call = calls[callId]; if (call != null) { call.onCandidatesReceived(event.content['candidates']); } else { Logs().v('[VOIP] onCallCandidates: Session [$callId] not found!'); } } void onCallHangup(Event event) async { // stop play ringtone, if this is an incoming call if (!delegate.isBackgroud) { delegate.stopRingtone(); } Logs().v('[VOIP] onCallHangup => ${event.content.toString()}'); final String callId = event.content['call_id']; final call = calls[callId]; if (call != null) { // hangup in any case, either if the other party hung up or we did on another device call.terminate(CallParty.kRemote, event.content['reason'] ?? CallErrorCode.UserHangup, true); } else { Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); } currentCID = null; } void onCallReject(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('Reject received for call ID ' + callId); final call = calls[callId]; if (call != null) { call.onRejectReceived(event.content['reason']); } else { Logs().v('[VOIP] onCallHangup: Session [$callId] not found!'); } } void onCallReplaces(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('onCallReplaces received for call ID ' + callId); final call = calls[callId]; if (call != null) { //TODO: handle replaces } } void onCallSelectAnswer(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('SelectAnswer received for call ID ' + callId); final call = calls[callId]; final String selectedPartyId = event.content['selected_party_id']; if (call != null) { call.onSelectAnswerReceived(selectedPartyId); } } void onSDPStreamMetadataChangedReceived(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('SDP Stream metadata received for call ID ' + callId); final call = calls[callId]; if (call != null) { if (event.content[sdpStreamMetadataKey] == null) { Logs().d('SDP Stream metadata is null'); return; } call.onSDPStreamMetadataReceived( SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey])); } } void onAssertedIdentityReceived(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('Asserted identity received for call ID ' + callId); final call = calls[callId]; if (call != null) { if (event.content['asserted_identity'] == null) { Logs().d('asserted_identity is null '); return; } call.onAssertedIdentityReceived( AssertedIdentity.fromJson(event.content['asserted_identity'])); } } void onCallNegotiate(Event event) async { if (event.senderId == client.userID) { // Ignore messages to yourself. return; } final String callId = event.content['call_id']; Logs().d('Negotiate received for call ID ' + callId); final call = calls[callId]; if (call != null) { final description = event.content['description']; try { SDPStreamMetadata? metadata; if (event.content[sdpStreamMetadataKey] != null) { metadata = SDPStreamMetadata.fromJson(event.content[sdpStreamMetadataKey]); } call.onNegotiateReceived(metadata, RTCSessionDescription(description['sdp'], description['type'])); } catch (err) { Logs().e('Failed to complete negotiation ${err.toString()}'); } } } CallType getCallType(String sdp) { try { final session = sdp_transform.parse(sdp); if (session['media'].indexWhere((e) => e['type'] == 'video') != -1) { return CallType.kVideo; } } catch (err) { Logs().e('Failed to getCallType ${err.toString()}'); } return CallType.kVoice; } Future requestTurnServerCredentials() async { return true; } Future>> getIceSevers() async { if (_turnServerCredentials == null) { try { _turnServerCredentials = await client.getTurnServer(); } catch (e) { Logs().v('[VOIP] getTurnServerCredentials error => ${e.toString()}'); } } if (_turnServerCredentials == null) { return []; } return [ { 'username': _turnServerCredentials!.username, 'credential': _turnServerCredentials!.password, 'urls': _turnServerCredentials!.uris[0] } ]; } Future inviteToCall(String roomId, CallType type) async { final room = client.getRoomById(roomId); if (room == null) { Logs().v('[VOIP] Invalid room id [$roomId].'); return Null as CallSession; } final callId = 'cid${DateTime.now().millisecondsSinceEpoch}'; final opts = CallOptions() ..callId = callId ..type = type ..dir = CallDirection.kOutgoing ..room = room ..voip = this ..localPartyId = localPartyId! ..iceServers = await getIceSevers(); final newCall = createNewCall(opts); currentCID = callId; await newCall.initOutboundCall(type).then((_) { if (!delegate.isBackgroud) { delegate.handleNewCall(newCall); } }); currentCID = callId; return newCall; } CallSession createNewCall(CallOptions opts) { final call = CallSession(opts); calls[opts.callId] = call; return call; } }