diff --git a/package.json b/package.json index b24de64..6a077d0 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "^1.9.7", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#af10b0c44b4a427c8d2224bfd6feb03a12cfd27e", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/src/livekit/useLiveKit.ts b/src/livekit/useLiveKit.ts index 4bd68a6..b888b05 100644 --- a/src/livekit/useLiveKit.ts +++ b/src/livekit/useLiveKit.ts @@ -1,5 +1,10 @@ -import { LocalAudioTrack, LocalVideoTrack, Room } from "livekit-client"; -import React from "react"; +import { + ConnectionState, + LocalAudioTrack, + LocalVideoTrack, + Room, +} from "livekit-client"; +import React, { useEffect } from "react"; import { useMediaDeviceSelect, usePreviewDevice, @@ -8,6 +13,7 @@ import { import { MediaDevicesState, MediaDevices } from "../settings/mediaDevices"; import { LocalMediaInfo, MediaInfo } from "../room/VideoPreview"; import { roomOptions } from "./options"; +import { useDefaultDevices } from "../settings/useSetting"; type LiveKitState = { // The state of the media devices (changing the devices will also change them in the room). @@ -19,6 +25,10 @@ type LiveKitState = { room: Room; }; +function emptyToUndef(str) { + return str === "" ? undefined : str; +} + // Returns the React state for the LiveKit's Room class. // The actual return type should be `LiveKitState`, but since this is a React hook, the initialisation is // delayed (done after the rendering, not during the rendering), because of that this function may return `undefined`. @@ -32,21 +42,35 @@ export function useLiveKit(): LiveKitState | undefined { // Create a React state to store the available devices and the selected device for each kind. const mediaDevices = useMediaDevicesState(room); - // Create local video track. + const [settingsDefaultDevices] = useDefaultDevices(); + const [videoEnabled, setVideoEnabled] = React.useState(true); const selectedVideoId = mediaDevices.state.get("videoinput")?.selectedId; + + const [audioEnabled, setAudioEnabled] = React.useState(true); + const selectedAudioId = mediaDevices.state.get("audioinput")?.selectedId; + + // trigger permission popup first, + useEffect(() => { + navigator.mediaDevices.getUserMedia({ + video: { deviceId: selectedVideoId ?? settingsDefaultDevices.videoinput }, + audio: { deviceId: selectedAudioId ?? settingsDefaultDevices.audioinput }, + }); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + // then start the preview device (no permsssion should be triggered agian) + // Create local video track. const video = usePreviewDevice( videoEnabled, - selectedVideoId ?? "", + selectedVideoId ?? settingsDefaultDevices.videoinput, "videoinput" ); // Create local audio track. - const [audioEnabled, setAudioEnabled] = React.useState(true); - const selectedAudioId = mediaDevices.state.get("audioinput")?.selectedId; const audio = usePreviewDevice( audioEnabled, - selectedAudioId ?? "", + selectedAudioId ?? settingsDefaultDevices.audioinput, "audioinput" ); @@ -71,7 +95,6 @@ export function useLiveKit(): LiveKitState | undefined { }, }; }; - const state: LiveKitState = { mediaDevices: mediaDevices, localMedia: { @@ -102,19 +125,24 @@ export function useLiveKit(): LiveKitState | undefined { return state; } +// if a room is passed this only affects the device selection inside a call. Without room it changes what we see in the lobby function useMediaDevicesState(room: Room): MediaDevicesState { + let connectedRoom: Room; + if (room.state !== ConnectionState.Disconnected) { + connectedRoom = room; + } const { devices: videoDevices, activeDeviceId: activeVideoDevice, setActiveMediaDevice: setActiveVideoDevice, - } = useMediaDeviceSelect({ kind: "videoinput", room }); + } = useMediaDeviceSelect({ kind: "videoinput", room: connectedRoom }); const { devices: audioDevices, activeDeviceId: activeAudioDevice, setActiveMediaDevice: setActiveAudioDevice, } = useMediaDeviceSelect({ kind: "audioinput", - room, + room: connectedRoom, }); const { devices: audioOutputDevices, @@ -122,7 +150,7 @@ function useMediaDevicesState(room: Room): MediaDevicesState { setActiveMediaDevice: setActiveAudioOutputDevice, } = useMediaDeviceSelect({ kind: "audiooutput", - room, + room: connectedRoom, }); const selectActiveDevice = React.useCallback( @@ -139,7 +167,7 @@ function useMediaDevicesState(room: Room): MediaDevicesState { break; } }, - [setActiveAudioDevice, setActiveVideoDevice, setActiveAudioOutputDevice] + [setActiveVideoDevice, setActiveAudioOutputDevice, setActiveAudioDevice] ); const [mediaDevicesState, setMediaDevicesState] = @@ -151,19 +179,35 @@ function useMediaDevicesState(room: Room): MediaDevicesState { return state; }); + const [settingsDefaultDevices, setDefaultDevices] = useDefaultDevices(); + React.useEffect(() => { const state = new Map(); state.set("videoinput", { available: videoDevices, - selectedId: activeVideoDevice, + selectedId: + emptyToUndef(activeVideoDevice) ?? + emptyToUndef(settingsDefaultDevices.videoinput) ?? + videoDevices[0]?.deviceId, }); state.set("audioinput", { available: audioDevices, - selectedId: activeAudioDevice, + selectedId: + emptyToUndef(activeAudioDevice) ?? + emptyToUndef(settingsDefaultDevices.audioinput) ?? + audioDevices[0]?.deviceId, }); state.set("audiooutput", { available: audioOutputDevices, - selectedId: activeAudioOutputDevice, + selectedId: + emptyToUndef(activeAudioOutputDevice) ?? + emptyToUndef(settingsDefaultDevices.audiooutput) ?? + audioOutputDevices[0]?.deviceId, + }); + setDefaultDevices({ + audioinput: state.get("audioinput").selectedId, + videoinput: state.get("videoinput").selectedId, + audiooutput: state.get("audiooutput").selectedId, }); setMediaDevicesState({ state, @@ -177,6 +221,10 @@ function useMediaDevicesState(room: Room): MediaDevicesState { audioOutputDevices, activeAudioOutputDevice, selectActiveDevice, + setDefaultDevices, + settingsDefaultDevices.audioinput, + settingsDefaultDevices.videoinput, + settingsDefaultDevices.audiooutput, ]); return mediaDevicesState; diff --git a/src/matrix-utils.ts b/src/matrix-utils.ts index e4f8388..9a7645d 100644 --- a/src/matrix-utils.ts +++ b/src/matrix-utils.ts @@ -173,6 +173,7 @@ export async function initClient( localTimeoutMs: 5000, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: fallbackICEServerAllowed, + useLivekitForGroupCalls: true, }); try { @@ -335,7 +336,8 @@ export async function createRoom( result.room_id, ptt ? GroupCallType.Voice : GroupCallType.Video, ptt, - GroupCallIntent.Room + GroupCallIntent.Room, + true ); return [fullAliasFromRoomName(name, client), result.room_id]; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2ec28e6..d1ca2d3 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -16,7 +16,6 @@ limitations under the License. import { ResizeObserver } from "@juggle/resize-observer"; import { - useLiveKitRoom, useLocalParticipant, useParticipants, useToken, @@ -78,6 +77,7 @@ import { InviteModal } from "./InviteModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { VideoTile } from "../video-grid/VideoTile"; +import { useRoom } from "./useRoom"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -85,6 +85,16 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // For now we can disable screensharing in Safari. const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +const onConnectedCallback = (): void => { + console.log("connected to LiveKit room"); +}; +const onDisconnectedCallback = (): void => { + console.log("disconnected from LiveKit room"); +}; +const onErrorCallback = (err: Error): void => { + console.error("error connecting to LiveKit room", err); +}; + interface LocalUserChoices { videoMuted: boolean; audioMuted: boolean; @@ -149,22 +159,23 @@ export function InCallView({ options ); + // TODO: move the room creation into the useRoom hook and out of the useLiveKit hook. + // This would than allow to not have those 4 lines + livekitRoom.options.audioCaptureDefaults.deviceId = + mediaDevices.state.get("audioinput").selectedId; + livekitRoom.options.videoCaptureDefaults.deviceId = + mediaDevices.state.get("videoinput").selectedId; // Uses a hook to connect to the LiveKit room (on unmount the room will be left) and publish local media tracks (default). - useLiveKitRoom({ + useRoom({ token, serverUrl: Config.get().livekit.server_url, room: livekitRoom, audio: !userChoices.audioMuted, video: !userChoices.videoMuted, - onConnected: () => { - console.log("connected to LiveKit room"); - }, - onDisconnected: () => { - console.log("disconnected from LiveKit room"); - }, - onError: (err) => { - console.error("error connecting to LiveKit room", err); - }, + simulateParticipants: 10, + onConnected: onConnectedCallback, + onDisconnected: onDisconnectedCallback, + onError: onErrorCallback, }); const screenSharingTracks = useTracks( @@ -453,6 +464,10 @@ function useParticipantTiles( }) ); + const someoneIsPresenting = sfuParticipants.some((p) => { + !p.isLocal && p.isScreenShareEnabled; + }); + // Iterate over SFU participants (those who actually are present from the SFU perspective) and create tiles for them. const tiles: TileDescriptor[] = sfuParticipants.flatMap( (sfuParticipant) => { @@ -461,7 +476,7 @@ function useParticipantTiles( const userMediaTile = { id, - focused: false, + focused: !someoneIsPresenting && sfuParticipant.isSpeaking, local: sfuParticipant.isLocal, data: { member, diff --git a/src/room/useRoom.ts b/src/room/useRoom.ts new file mode 100644 index 0000000..63bb153 --- /dev/null +++ b/src/room/useRoom.ts @@ -0,0 +1,135 @@ +import * as React from "react"; +import { + ConnectionState, + MediaDeviceFailure, + Room, + RoomEvent, +} from "livekit-client"; +import { LiveKitRoomProps } from "@livekit/components-react/src/components/LiveKitRoom"; +import { logger } from "matrix-js-sdk/src/logger"; + +const defaultRoomProps: Partial = { + connect: true, + audio: false, + video: false, +}; + +export function useRoom(props: LiveKitRoomProps) { + const { + token, + serverUrl, + options, + room: passedRoom, + connectOptions, + connect, + audio, + video, + screen, + onConnected, + onDisconnected, + onError, + onMediaDeviceFailure, + } = { ...defaultRoomProps, ...props }; + if (options && passedRoom) { + logger.warn( + "when using a manually created room, the options object will be ignored. set the desired options directly when creating the room instead." + ); + } + + const [room, setRoom] = React.useState(); + + React.useEffect(() => { + setRoom(passedRoom ?? new Room(options)); + }, [options, passedRoom]); + + React.useEffect(() => { + if (!room) return; + const onSignalConnected = () => { + const localP = room.localParticipant; + try { + logger.debug("trying to publish local tracks"); + localP.setMicrophoneEnabled( + !!audio, + typeof audio !== "boolean" ? audio : undefined + ); + localP.setCameraEnabled( + !!video, + typeof video !== "boolean" ? video : undefined + ); + localP.setScreenShareEnabled( + !!screen, + typeof screen !== "boolean" ? screen : undefined + ); + } catch (e) { + logger.warn(e); + onError?.(e as Error); + } + }; + + const onMediaDeviceError = (e: Error) => { + const mediaDeviceFailure = MediaDeviceFailure.getFailure(e); + onMediaDeviceFailure?.(mediaDeviceFailure); + }; + room.on(RoomEvent.SignalConnected, onSignalConnected); + room.on(RoomEvent.MediaDevicesError, onMediaDeviceError); + + return () => { + room.off(RoomEvent.SignalConnected, onSignalConnected); + room.off(RoomEvent.MediaDevicesError, onMediaDeviceError); + }; + }, [room, audio, video, screen, onError, onMediaDeviceFailure]); + + React.useEffect(() => { + if (!room) return; + if (!token) { + logger.debug("no token yet"); + return; + } + if (!serverUrl) { + logger.warn("no livekit url provided"); + onError?.(Error("no livekit url provided")); + return; + } + if (connect) { + logger.debug("connecting"); + room.connect(serverUrl, token, connectOptions).catch((e) => { + logger.warn(e); + onError?.(e as Error); + }); + } else { + logger.debug("disconnecting because connect is false"); + room.disconnect(); + } + }, [connect, token, connectOptions, room, onError, serverUrl]); + + React.useEffect(() => { + if (!room) return; + const connectionStateChangeListener = (state: ConnectionState) => { + switch (state) { + case ConnectionState.Disconnected: + if (onDisconnected) onDisconnected(); + break; + case ConnectionState.Connected: + if (onConnected) onConnected(); + break; + + default: + break; + } + }; + room.on(RoomEvent.ConnectionStateChanged, connectionStateChangeListener); + return () => { + room.off(RoomEvent.ConnectionStateChanged, connectionStateChangeListener); + }; + }, [token, onConnected, onDisconnected, room]); + + React.useEffect(() => { + if (!room) return; + return () => { + logger.info("disconnecting on onmount"); + room.disconnect(); + }; + }, [room]); + + return { room }; +} diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts index 8d0c5a0..b862c21 100644 --- a/src/settings/useSetting.ts +++ b/src/settings/useSetting.ts @@ -100,3 +100,10 @@ export const useOptInAnalytics = (): DisableableSetting => { export const useDeveloperSettingsTab = () => useSetting("developer-settings-tab", false); + +export const useDefaultDevices = () => + useSetting("defaultDevices", { + audioinput: "", + videoinput: "", + audiooutput: "", + }); diff --git a/yarn.lock b/yarn.lock index 94322cd..244dd11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1860,10 +1860,10 @@ "@react-hook/latest" "^1.0.3" clsx "^1.2.1" -"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.9": - version "0.1.0-alpha.9" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.9.tgz#00bc266781502641a661858a5a521dd4d95275fc" - integrity sha512-g5cjpFwA9h0CbEGoAqNVI2QcyDsbI8FHoLo9+OXWHIezEKITsSv78mc5ilIwN+2YpmVlH0KNeQWTHw4vi0BMnw== +"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.10": + version "0.1.0-alpha.10" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.10.tgz#b6a6395cffd3197ae2e0a88f4eeae8b315571fd2" + integrity sha512-8V2NKuzGOFzEZeZVgF2is7gmuopdRbMZ064tzPDE0vN34iX6s3O8A4oxIT7SA3qtymwm3t1yEvTnT+0gfbmh4g== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14" @@ -10699,12 +10699,12 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1": - version "26.0.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#af10b0c44b4a427c8d2224bfd6feb03a12cfd27e": + version "26.0.1" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/af10b0c44b4a427c8d2224bfd6feb03a12cfd27e" dependencies: "@babel/runtime" "^7.12.5" - "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9" + "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.10" another-json "^0.2.0" bs58 "^5.0.0" content-type "^1.0.4"