From 30e1034fba6bca9bedd51a1f1c937f5019a37971 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 14 Jun 2023 12:13:55 +0100 Subject: [PATCH 1/5] Use new-js-sdk flag to disable the actual calling of group calls In favour of using livekit --- package.json | 2 +- src/matrix-utils.ts | 4 +++- yarn.lock | 16 ++++++++-------- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index b24de64..6a077d0 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "^1.9.7", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#af10b0c44b4a427c8d2224bfd6feb03a12cfd27e", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/src/matrix-utils.ts b/src/matrix-utils.ts index e4f8388..9a7645d 100644 --- a/src/matrix-utils.ts +++ b/src/matrix-utils.ts @@ -173,6 +173,7 @@ export async function initClient( localTimeoutMs: 5000, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: fallbackICEServerAllowed, + useLivekitForGroupCalls: true, }); try { @@ -335,7 +336,8 @@ export async function createRoom( result.room_id, ptt ? GroupCallType.Voice : GroupCallType.Video, ptt, - GroupCallIntent.Room + GroupCallIntent.Room, + true ); return [fullAliasFromRoomName(name, client), result.room_id]; diff --git a/yarn.lock b/yarn.lock index 94322cd..244dd11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1860,10 +1860,10 @@ "@react-hook/latest" "^1.0.3" clsx "^1.2.1" -"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.9": - version "0.1.0-alpha.9" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.9.tgz#00bc266781502641a661858a5a521dd4d95275fc" - integrity sha512-g5cjpFwA9h0CbEGoAqNVI2QcyDsbI8FHoLo9+OXWHIezEKITsSv78mc5ilIwN+2YpmVlH0KNeQWTHw4vi0BMnw== +"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.10": + version "0.1.0-alpha.10" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.10.tgz#b6a6395cffd3197ae2e0a88f4eeae8b315571fd2" + integrity sha512-8V2NKuzGOFzEZeZVgF2is7gmuopdRbMZ064tzPDE0vN34iX6s3O8A4oxIT7SA3qtymwm3t1yEvTnT+0gfbmh4g== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14" @@ -10699,12 +10699,12 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1": - version "26.0.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#af10b0c44b4a427c8d2224bfd6feb03a12cfd27e": + version "26.0.1" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/af10b0c44b4a427c8d2224bfd6feb03a12cfd27e" dependencies: "@babel/runtime" "^7.12.5" - "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9" + "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.10" another-json "^0.2.0" bs58 "^5.0.0" content-type "^1.0.4" From 20c9c092586983e206a37576a23346e9dc012aea Mon Sep 17 00:00:00 2001 From: Enrico Schwendig Date: Wed, 14 Jun 2023 15:03:38 +0200 Subject: [PATCH 2/5] Build EC room use method (#1108) * Build custom useRoom --- src/room/InCallView.tsx | 28 +++++---- src/room/useRoom.ts | 135 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 src/room/useRoom.ts diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d9005c0..5b7c000 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -16,7 +16,6 @@ limitations under the License. import { ResizeObserver } from "@juggle/resize-observer"; import { - useLiveKitRoom, useLocalParticipant, useParticipants, useToken, @@ -79,6 +78,7 @@ import { InviteModal } from "./InviteModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { VideoTile } from "../video-grid/VideoTile"; +import { useRoom } from "./useRoom"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -86,6 +86,16 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // For now we can disable screensharing in Safari. const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +const onConnectedCallback = (): void => { + console.log("connected to LiveKit room"); +}; +const onDisconnectedCallback = (): void => { + console.log("disconnected from LiveKit room"); +}; +const onErrorCallback = (err: Error): void => { + console.error("error connecting to LiveKit room", err); +}; + interface LocalUserChoices { videoMuted: boolean; audioMuted: boolean; @@ -150,22 +160,16 @@ export function InCallView({ options ); - // Uses a hook to connect to the LiveKit room (on unmount the room will be left) and publish local media tracks (default). - useLiveKitRoom({ + useRoom({ token, serverUrl: Config.get().livekit.server_url, room: livekitRoom, audio: !userChoices.audioMuted, video: !userChoices.videoMuted, - onConnected: () => { - console.log("connected to LiveKit room"); - }, - onDisconnected: () => { - console.log("disconnected from LiveKit room"); - }, - onError: (err) => { - console.error("error connecting to LiveKit room", err); - }, + simulateParticipants: 10, + onConnected: onConnectedCallback, + onDisconnected: onDisconnectedCallback, + onError: onErrorCallback, }); const screenSharingTracks = useTracks( diff --git a/src/room/useRoom.ts b/src/room/useRoom.ts new file mode 100644 index 0000000..63bb153 --- /dev/null +++ b/src/room/useRoom.ts @@ -0,0 +1,135 @@ +import * as React from "react"; +import { + ConnectionState, + MediaDeviceFailure, + Room, + RoomEvent, +} from "livekit-client"; +import { LiveKitRoomProps } from "@livekit/components-react/src/components/LiveKitRoom"; +import { logger } from "matrix-js-sdk/src/logger"; + +const defaultRoomProps: Partial = { + connect: true, + audio: false, + video: false, +}; + +export function useRoom(props: LiveKitRoomProps) { + const { + token, + serverUrl, + options, + room: passedRoom, + connectOptions, + connect, + audio, + video, + screen, + onConnected, + onDisconnected, + onError, + onMediaDeviceFailure, + } = { ...defaultRoomProps, ...props }; + if (options && passedRoom) { + logger.warn( + "when using a manually created room, the options object will be ignored. set the desired options directly when creating the room instead." + ); + } + + const [room, setRoom] = React.useState(); + + React.useEffect(() => { + setRoom(passedRoom ?? new Room(options)); + }, [options, passedRoom]); + + React.useEffect(() => { + if (!room) return; + const onSignalConnected = () => { + const localP = room.localParticipant; + try { + logger.debug("trying to publish local tracks"); + localP.setMicrophoneEnabled( + !!audio, + typeof audio !== "boolean" ? audio : undefined + ); + localP.setCameraEnabled( + !!video, + typeof video !== "boolean" ? video : undefined + ); + localP.setScreenShareEnabled( + !!screen, + typeof screen !== "boolean" ? screen : undefined + ); + } catch (e) { + logger.warn(e); + onError?.(e as Error); + } + }; + + const onMediaDeviceError = (e: Error) => { + const mediaDeviceFailure = MediaDeviceFailure.getFailure(e); + onMediaDeviceFailure?.(mediaDeviceFailure); + }; + room.on(RoomEvent.SignalConnected, onSignalConnected); + room.on(RoomEvent.MediaDevicesError, onMediaDeviceError); + + return () => { + room.off(RoomEvent.SignalConnected, onSignalConnected); + room.off(RoomEvent.MediaDevicesError, onMediaDeviceError); + }; + }, [room, audio, video, screen, onError, onMediaDeviceFailure]); + + React.useEffect(() => { + if (!room) return; + if (!token) { + logger.debug("no token yet"); + return; + } + if (!serverUrl) { + logger.warn("no livekit url provided"); + onError?.(Error("no livekit url provided")); + return; + } + if (connect) { + logger.debug("connecting"); + room.connect(serverUrl, token, connectOptions).catch((e) => { + logger.warn(e); + onError?.(e as Error); + }); + } else { + logger.debug("disconnecting because connect is false"); + room.disconnect(); + } + }, [connect, token, connectOptions, room, onError, serverUrl]); + + React.useEffect(() => { + if (!room) return; + const connectionStateChangeListener = (state: ConnectionState) => { + switch (state) { + case ConnectionState.Disconnected: + if (onDisconnected) onDisconnected(); + break; + case ConnectionState.Connected: + if (onConnected) onConnected(); + break; + + default: + break; + } + }; + room.on(RoomEvent.ConnectionStateChanged, connectionStateChangeListener); + return () => { + room.off(RoomEvent.ConnectionStateChanged, connectionStateChangeListener); + }; + }, [token, onConnected, onDisconnected, room]); + + React.useEffect(() => { + if (!room) return; + return () => { + logger.info("disconnecting on onmount"); + room.disconnect(); + }; + }, [room]); + + return { room }; +} From 357e1f828aa6580aa8b3a8d4613179efd647e788 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 14 Jun 2023 11:25:31 -0400 Subject: [PATCH 3/5] Fix spotlight layout --- src/room/InCallView.tsx | 24 ++++++++++++++++++++---- src/video-grid/VideoTile.tsx | 1 - 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5b7c000..92761fe 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -23,7 +23,12 @@ import { } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; import classNames from "classnames"; -import { Room, Track } from "livekit-client"; +import { + LocalParticipant, + RemoteParticipant, + Room, + Track, +} from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; @@ -473,11 +478,19 @@ function useParticipantTiles( const items = useMemo(() => { const tiles: TileDescriptor[] = []; + const screenshareExists = sfuParticipants.some( + (p) => p.isScreenShareEnabled + ); + const participantsById = new Map< + string, + LocalParticipant | RemoteParticipant + >(); + for (const p of sfuParticipants) participantsById.set(p.identity, p); for (const [member, participantMap] of participants) { for (const [deviceId] of participantMap) { const id = `${member.userId}:${deviceId}`; - const sfuParticipant = sfuParticipants.find((p) => p.identity === id); + const sfuParticipant = participantsById.get(id); // Skip rendering participants that did not connect to the SFU. if (!sfuParticipant) { @@ -486,10 +499,13 @@ function useParticipantTiles( const userMediaTile = { id, - focused: false, + // Screenshare feeds take precedence for focus + focused: + !screenshareExists && + sfuParticipant.isSpeaking && + !sfuParticipant.isLocal, local: sfuParticipant.isLocal, data: { - id, member, sfuParticipant, content: TileContent.UserMedia, diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index beff7ec..0fc47ec 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -32,7 +32,6 @@ import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { useRoomMemberName } from "./useRoomMemberName"; export interface ItemData { - id: string; member: RoomMember; sfuParticipant: LocalParticipant | RemoteParticipant; content: TileContent; From 037495a63516391084332789e89809531b69e7a8 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 14 Jun 2023 11:55:43 -0400 Subject: [PATCH 4/5] Fix tiles having the wrong states during screensharing This fixes a couple bugs: 1. That muting your video while screensharing would cause the screensharing feed to be hidden as well 2. That while screensharing, your user media tile would incorrectly show the label that's supposed to appear only on the screenshare tile --- src/video-grid/VideoTile.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index beff7ec..c4ea475 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -101,14 +101,14 @@ export const VideoTile = forwardRef( ref={tileRef} data-testid="videoTile" > - {!sfuParticipant.isCameraEnabled && ( + {content === TileContent.UserMedia && !sfuParticipant.isCameraEnabled && ( <>
{getAvatar(data.member, targetWidth, targetHeight)} )} {!false && - (sfuParticipant.isScreenShareEnabled ? ( + (content === TileContent.ScreenShare ? (
{t("{{name}} is presenting", { name })}
From 41f27287242af5729479a0639ae1ac5255c6c9ab Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Wed, 14 Jun 2023 19:20:53 +0200 Subject: [PATCH 5/5] Device from lobby to call (#1110) * respect mute state set in lobby for call Signed-off-by: Timo K * move device from lobby to call Signed-off-by: Timo K * save device in local storage Signed-off-by: Timo K * local storage + fixes Signed-off-by: Timo K * device permissions Signed-off-by: Timo K --------- Signed-off-by: Timo K --- src/livekit/useLiveKit.ts | 78 ++++++++++++++++++++++++++++++-------- src/room/InCallView.tsx | 7 ++++ src/settings/useSetting.ts | 7 ++++ 3 files changed, 77 insertions(+), 15 deletions(-) diff --git a/src/livekit/useLiveKit.ts b/src/livekit/useLiveKit.ts index 4bd68a6..b888b05 100644 --- a/src/livekit/useLiveKit.ts +++ b/src/livekit/useLiveKit.ts @@ -1,5 +1,10 @@ -import { LocalAudioTrack, LocalVideoTrack, Room } from "livekit-client"; -import React from "react"; +import { + ConnectionState, + LocalAudioTrack, + LocalVideoTrack, + Room, +} from "livekit-client"; +import React, { useEffect } from "react"; import { useMediaDeviceSelect, usePreviewDevice, @@ -8,6 +13,7 @@ import { import { MediaDevicesState, MediaDevices } from "../settings/mediaDevices"; import { LocalMediaInfo, MediaInfo } from "../room/VideoPreview"; import { roomOptions } from "./options"; +import { useDefaultDevices } from "../settings/useSetting"; type LiveKitState = { // The state of the media devices (changing the devices will also change them in the room). @@ -19,6 +25,10 @@ type LiveKitState = { room: Room; }; +function emptyToUndef(str) { + return str === "" ? undefined : str; +} + // Returns the React state for the LiveKit's Room class. // The actual return type should be `LiveKitState`, but since this is a React hook, the initialisation is // delayed (done after the rendering, not during the rendering), because of that this function may return `undefined`. @@ -32,21 +42,35 @@ export function useLiveKit(): LiveKitState | undefined { // Create a React state to store the available devices and the selected device for each kind. const mediaDevices = useMediaDevicesState(room); - // Create local video track. + const [settingsDefaultDevices] = useDefaultDevices(); + const [videoEnabled, setVideoEnabled] = React.useState(true); const selectedVideoId = mediaDevices.state.get("videoinput")?.selectedId; + + const [audioEnabled, setAudioEnabled] = React.useState(true); + const selectedAudioId = mediaDevices.state.get("audioinput")?.selectedId; + + // trigger permission popup first, + useEffect(() => { + navigator.mediaDevices.getUserMedia({ + video: { deviceId: selectedVideoId ?? settingsDefaultDevices.videoinput }, + audio: { deviceId: selectedAudioId ?? settingsDefaultDevices.audioinput }, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // then start the preview device (no permsssion should be triggered agian) + // Create local video track. const video = usePreviewDevice( videoEnabled, - selectedVideoId ?? "", + selectedVideoId ?? settingsDefaultDevices.videoinput, "videoinput" ); // Create local audio track. - const [audioEnabled, setAudioEnabled] = React.useState(true); - const selectedAudioId = mediaDevices.state.get("audioinput")?.selectedId; const audio = usePreviewDevice( audioEnabled, - selectedAudioId ?? "", + selectedAudioId ?? settingsDefaultDevices.audioinput, "audioinput" ); @@ -71,7 +95,6 @@ export function useLiveKit(): LiveKitState | undefined { }, }; }; - const state: LiveKitState = { mediaDevices: mediaDevices, localMedia: { @@ -102,19 +125,24 @@ export function useLiveKit(): LiveKitState | undefined { return state; } +// if a room is passed this only affects the device selection inside a call. Without room it changes what we see in the lobby function useMediaDevicesState(room: Room): MediaDevicesState { + let connectedRoom: Room; + if (room.state !== ConnectionState.Disconnected) { + connectedRoom = room; + } const { devices: videoDevices, activeDeviceId: activeVideoDevice, setActiveMediaDevice: setActiveVideoDevice, - } = useMediaDeviceSelect({ kind: "videoinput", room }); + } = useMediaDeviceSelect({ kind: "videoinput", room: connectedRoom }); const { devices: audioDevices, activeDeviceId: activeAudioDevice, setActiveMediaDevice: setActiveAudioDevice, } = useMediaDeviceSelect({ kind: "audioinput", - room, + room: connectedRoom, }); const { devices: audioOutputDevices, @@ -122,7 +150,7 @@ function useMediaDevicesState(room: Room): MediaDevicesState { setActiveMediaDevice: setActiveAudioOutputDevice, } = useMediaDeviceSelect({ kind: "audiooutput", - room, + room: connectedRoom, }); const selectActiveDevice = React.useCallback( @@ -139,7 +167,7 @@ function useMediaDevicesState(room: Room): MediaDevicesState { break; } }, - [setActiveAudioDevice, setActiveVideoDevice, setActiveAudioOutputDevice] + [setActiveVideoDevice, setActiveAudioOutputDevice, setActiveAudioDevice] ); const [mediaDevicesState, setMediaDevicesState] = @@ -151,19 +179,35 @@ function useMediaDevicesState(room: Room): MediaDevicesState { return state; }); + const [settingsDefaultDevices, setDefaultDevices] = useDefaultDevices(); + React.useEffect(() => { const state = new Map(); state.set("videoinput", { available: videoDevices, - selectedId: activeVideoDevice, + selectedId: + emptyToUndef(activeVideoDevice) ?? + emptyToUndef(settingsDefaultDevices.videoinput) ?? + videoDevices[0]?.deviceId, }); state.set("audioinput", { available: audioDevices, - selectedId: activeAudioDevice, + selectedId: + emptyToUndef(activeAudioDevice) ?? + emptyToUndef(settingsDefaultDevices.audioinput) ?? + audioDevices[0]?.deviceId, }); state.set("audiooutput", { available: audioOutputDevices, - selectedId: activeAudioOutputDevice, + selectedId: + emptyToUndef(activeAudioOutputDevice) ?? + emptyToUndef(settingsDefaultDevices.audiooutput) ?? + audioOutputDevices[0]?.deviceId, + }); + setDefaultDevices({ + audioinput: state.get("audioinput").selectedId, + videoinput: state.get("videoinput").selectedId, + audiooutput: state.get("audiooutput").selectedId, }); setMediaDevicesState({ state, @@ -177,6 +221,10 @@ function useMediaDevicesState(room: Room): MediaDevicesState { audioOutputDevices, activeAudioOutputDevice, selectActiveDevice, + setDefaultDevices, + settingsDefaultDevices.audioinput, + settingsDefaultDevices.videoinput, + settingsDefaultDevices.audiooutput, ]); return mediaDevicesState; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 92761fe..611f130 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -165,6 +165,13 @@ export function InCallView({ options ); + // TODO: move the room creation into the useRoom hook and out of the useLiveKit hook. + // This would than allow to not have those 4 lines + livekitRoom.options.audioCaptureDefaults.deviceId = + mediaDevices.state.get("audioinput").selectedId; + livekitRoom.options.videoCaptureDefaults.deviceId = + mediaDevices.state.get("videoinput").selectedId; + // Uses a hook to connect to the LiveKit room (on unmount the room will be left) and publish local media tracks (default). useRoom({ token, serverUrl: Config.get().livekit.server_url, diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts index 8d0c5a0..b862c21 100644 --- a/src/settings/useSetting.ts +++ b/src/settings/useSetting.ts @@ -100,3 +100,10 @@ export const useOptInAnalytics = (): DisableableSetting => { export const useDeveloperSettingsTab = () => useSetting("developer-settings-tab", false); + +export const useDefaultDevices = () => + useSetting("defaultDevices", { + audioinput: "", + videoinput: "", + audiooutput: "", + });