From 1ff9073a1afc038fd0913d25dd3b68eb2323a4fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 May 2022 12:14:25 +0100 Subject: [PATCH 01/15] Sort call feeds consistently when choosing active speaker --- src/room/usePTT.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts index 5965c54..cc42140 100644 --- a/src/room/usePTT.ts +++ b/src/room/usePTT.ts @@ -30,6 +30,17 @@ function getActiveSpeakerFeed( ): CallFeed | null { const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted()); + // make sure the feeds are in a deterministic order so every client picks + // the same one as the active speaker + const collator = new Intl.Collator("en", { + sensitivity: "variant", + usage: "sort", + ignorePunctuation: false, + }); + activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number => + collator.compare(a.userId, b.userId) + ); + let activeSpeakerFeed = null; let highestPowerLevel = null; for (const feed of activeSpeakerFeeds) { From 21c7bb979ee1ade3bfb18b1435ea1960dd45c282 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 May 2022 15:30:57 +0100 Subject: [PATCH 02/15] Convert useGroupCall to TS --- src/room/{useGroupCall.js => useGroupCall.ts} | 59 ++++++++++++++----- 1 file changed, 45 insertions(+), 14 deletions(-) rename src/room/{useGroupCall.js => useGroupCall.ts} (82%) diff --git a/src/room/useGroupCall.js b/src/room/useGroupCall.ts similarity index 82% rename from src/room/useGroupCall.js rename to src/room/useGroupCall.ts index 51900c5..25c2179 100644 --- a/src/room/useGroupCall.js +++ b/src/room/useGroupCall.ts @@ -18,10 +18,33 @@ import { useCallback, useEffect, useState } from "react"; import { GroupCallEvent, GroupCallState, + GroupCall, } from "matrix-js-sdk/src/webrtc/groupCall"; +import { MatrixCall } from "matrix-js-sdk/src/webrtc/call"; +import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; +import { RoomMember } from "matrix-js-sdk/src/models/room-member"; + import { usePageUnload } from "./usePageUnload"; -export function useGroupCall(groupCall) { +interface State { + state: GroupCallState; + calls: MatrixCall[]; + localCallFeed: CallFeed; + activeSpeaker: string; + userMediaFeeds: CallFeed[]; + error: Error; + microphoneMuted: boolean; + localVideoMuted: boolean; + screenshareFeeds: CallFeed[]; + localScreenshareFeed: CallFeed; + localDesktopCapturerSourceId: string; + isScreensharing: boolean; + requestingScreenshare: boolean; + participants: RoomMember[]; + hasLocalParticipant: boolean; +} + +export function useGroupCall(groupCall: GroupCall) { const [ { state, @@ -41,20 +64,25 @@ export function useGroupCall(groupCall) { requestingScreenshare, }, setState, - ] = useState({ + ] = useState({ state: GroupCallState.LocalCallFeedUninitialized, calls: [], + localCallFeed: null, + activeSpeaker: null, userMediaFeeds: [], + error: null, microphoneMuted: false, localVideoMuted: false, - screenshareFeeds: [], isScreensharing: false, + screenshareFeeds: [], + localScreenshareFeed: null, + localDesktopCapturerSourceId: null, requestingScreenshare: false, participants: [], hasLocalParticipant: false, }); - const updateState = (state) => + const updateState = (state: Partial) => setState((prevState) => ({ ...prevState, ...state })); useEffect(() => { @@ -75,25 +103,28 @@ export function useGroupCall(groupCall) { }); } - function onUserMediaFeedsChanged(userMediaFeeds) { + function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void { updateState({ userMediaFeeds: [...userMediaFeeds], }); } - function onScreenshareFeedsChanged(screenshareFeeds) { + function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void { updateState({ screenshareFeeds: [...screenshareFeeds], }); } - function onActiveSpeakerChanged(activeSpeaker) { + function onActiveSpeakerChanged(activeSpeaker: string): void { updateState({ activeSpeaker: activeSpeaker, }); } - function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) { + function onLocalMuteStateChanged( + microphoneMuted: boolean, + localVideoMuted: boolean + ): void { updateState({ microphoneMuted, localVideoMuted, @@ -101,10 +132,10 @@ export function useGroupCall(groupCall) { } function onLocalScreenshareStateChanged( - isScreensharing, - localScreenshareFeed, - localDesktopCapturerSourceId - ) { + isScreensharing: boolean, + localScreenshareFeed: CallFeed, + localDesktopCapturerSourceId: string + ): void { updateState({ isScreensharing, localScreenshareFeed, @@ -112,13 +143,13 @@ export function useGroupCall(groupCall) { }); } - function onCallsChanged(calls) { + function onCallsChanged(calls: MatrixCall[]): void { updateState({ calls: [...calls], }); } - function onParticipantsChanged(participants) { + function onParticipantsChanged(participants: RoomMember[]): void { updateState({ participants: [...participants], hasLocalParticipant: groupCall.hasLocalParticipant(), From 1164e6f1e7282e394a2ed7409f795e73707cff2d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 May 2022 15:53:44 +0100 Subject: [PATCH 03/15] Add return type too --- src/room/useGroupCall.ts | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts index 25c2179..6816bcf 100644 --- a/src/room/useGroupCall.ts +++ b/src/room/useGroupCall.ts @@ -26,6 +26,30 @@ import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { usePageUnload } from "./usePageUnload"; +export interface UseGroupCallType { + state: GroupCallState; + calls: MatrixCall[]; + localCallFeed: CallFeed; + activeSpeaker: string; + userMediaFeeds: CallFeed[]; + microphoneMuted: boolean; + localVideoMuted: boolean; + error: Error; + initLocalCallFeed: () => void; + enter: () => void; + leave: () => void; + toggleLocalVideoMuted: () => void; + toggleMicrophoneMuted: () => void; + toggleScreensharing: () => void; + requestingScreenshare: boolean; + isScreensharing: boolean; + screenshareFeeds: CallFeed[]; + localScreenshareFeed: CallFeed; + localDesktopCapturerSourceId: string; + participants: RoomMember[]; + hasLocalParticipant: boolean; +} + interface State { state: GroupCallState; calls: MatrixCall[]; @@ -44,7 +68,7 @@ interface State { hasLocalParticipant: boolean; } -export function useGroupCall(groupCall: GroupCall) { +export function useGroupCall(groupCall: GroupCall): UseGroupCallType { const [ { state, From e9b963080cd3ddcd8829a21983ef2b40db9995f0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 30 May 2022 16:28:16 +0100 Subject: [PATCH 04/15] Show when connection is lost on PTT mode --- src/room/PTTCallView.tsx | 9 +++++++-- src/room/usePTT.ts | 26 +++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx index 4b395f8..12157d2 100644 --- a/src/room/PTTCallView.tsx +++ b/src/room/PTTCallView.tsx @@ -44,8 +44,11 @@ function getPromptText( activeSpeakerIsLocalUser: boolean, talkOverEnabled: boolean, activeSpeakerUserId: string, - activeSpeakerDisplayName: string + activeSpeakerDisplayName: string, + connected: boolean ): string { + if (!connected) return "Connection Lost"; + const isTouchScreen = Boolean(window.ontouchstart !== undefined); if (showTalkOverError) { @@ -127,6 +130,7 @@ export const PTTCallView: React.FC = ({ startTalking, stopTalking, transmitBlocked, + connected, } = usePTT( client, groupCall, @@ -234,7 +238,8 @@ export const PTTCallView: React.FC = ({ activeSpeakerIsLocalUser, talkOverEnabled, activeSpeakerUserId, - activeSpeakerDisplayName + activeSpeakerDisplayName, + connected )}

{userMediaFeeds.map((callFeed) => ( diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts index 5965c54..328ea44 100644 --- a/src/room/usePTT.ts +++ b/src/room/usePTT.ts @@ -15,10 +15,11 @@ limitations under the License. */ import { useCallback, useEffect, useState } from "react"; -import { MatrixClient } from "matrix-js-sdk/src/client"; +import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client"; import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed"; import { logger } from "matrix-js-sdk/src/logger"; +import { SyncState } from "matrix-js-sdk/src/sync"; import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds"; @@ -52,6 +53,11 @@ export interface PTTState { startTalking: () => void; stopTalking: () => void; transmitBlocked: boolean; + // connected is actually an indication of whether we're connected to the HS + // (ie. the client's syncing state) rather than media connection, since + // it's peer to peer so we can't really say once peer is 'disconnected' if + // there's only one other person in the call and they've lost Internet. + connected: boolean; } export const usePTT = ( @@ -211,6 +217,17 @@ export const usePTT = ( setMicMuteWrapper(true); }, [setMicMuteWrapper]); + // separate state for connected: we set it separately from other things + // in the client sync callback + const [connected, setConnected] = useState(true); + + const onClientSync = useCallback( + (syncState: SyncState) => { + setConnected(syncState !== SyncState.Error); + }, + [setConnected] + ); + useEffect(() => { function onKeyDown(event: KeyboardEvent): void { if (event.code === "Space") { @@ -245,10 +262,14 @@ export const usePTT = ( window.addEventListener("keyup", onKeyUp); window.addEventListener("blur", onBlur); + client.on(ClientEvent.Sync, onClientSync); + return () => { window.removeEventListener("keydown", onKeyDown); window.removeEventListener("keyup", onKeyUp); window.removeEventListener("blur", onBlur); + + client.removeListener(ClientEvent.Sync, onClientSync); }; }, [ groupCall, @@ -260,6 +281,8 @@ export const usePTT = ( pttButtonHeld, enablePTTButton, setMicMuteWrapper, + client, + onClientSync, ]); const setTalkOverEnabled = useCallback((talkOverEnabled) => { @@ -278,5 +301,6 @@ export const usePTT = ( startTalking, stopTalking, transmitBlocked, + connected, }; }; From c6b90803f8e8552733f670bbd5e04d7cf7fa8bfc Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Tue, 31 May 2022 10:43:05 -0400 Subject: [PATCH 05/15] Add spatial audio capabilities --- src/room/GroupCallView.jsx | 19 ----- src/room/InCallView.jsx | 12 ++-- src/room/LobbyView.jsx | 4 -- src/room/OverflowMenu.jsx | 10 +-- src/room/PTTCallView.tsx | 6 -- src/room/VideoPreview.jsx | 4 -- src/settings/SettingsModal.jsx | 18 ++++- src/settings/useSetting.ts | 56 +++++++++++++++ src/video-grid/VideoTile.jsx | 100 ++++++++++++++------------ src/video-grid/VideoTile.module.css | 4 ++ src/video-grid/VideoTileContainer.jsx | 11 ++- src/video-grid/useMediaStream.js | 60 ++++++++++++++++ 12 files changed, 205 insertions(+), 99 deletions(-) create mode 100644 src/settings/useSetting.ts diff --git a/src/room/GroupCallView.jsx b/src/room/GroupCallView.jsx index dcd9da3..2809b82 100644 --- a/src/room/GroupCallView.jsx +++ b/src/room/GroupCallView.jsx @@ -33,19 +33,6 @@ export function GroupCallView({ roomId, groupCall, }) { - const [showInspector, setShowInspector] = useState( - () => !!localStorage.getItem("matrix-group-call-inspector") - ); - const onChangeShowInspector = useCallback((show) => { - setShowInspector(show); - - if (show) { - localStorage.setItem("matrix-group-call-inspector", "true"); - } else { - localStorage.removeItem("matrix-group-call-inspector"); - } - }, []); - const { state, error, @@ -104,8 +91,6 @@ export function GroupCallView({ participants={participants} userMediaFeeds={userMediaFeeds} onLeave={onLeave} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} /> ); } else { @@ -126,8 +111,6 @@ export function GroupCallView({ isScreensharing={isScreensharing} localScreenshareFeed={localScreenshareFeed} screenshareFeeds={screenshareFeeds} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} roomId={roomId} /> ); @@ -156,8 +139,6 @@ export function GroupCallView({ localVideoMuted={localVideoMuted} toggleLocalVideoMuted={toggleLocalVideoMuted} toggleMicrophoneMuted={toggleMicrophoneMuted} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} roomId={roomId} /> ); diff --git a/src/room/InCallView.jsx b/src/room/InCallView.jsx index a474963..2c6240e 100644 --- a/src/room/InCallView.jsx +++ b/src/room/InCallView.jsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { useCallback, useMemo } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import styles from "./InCallView.module.css"; import { HangupButton, @@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { usePreventScroll } from "@react-aria/overlays"; import { useMediaHandler } from "../settings/useMediaHandler"; +import { useShowInspector } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; const canScreenshare = "getDisplayMedia" in navigator.mediaDevices; @@ -57,14 +58,16 @@ export function InCallView({ toggleScreensharing, isScreensharing, screenshareFeeds, - setShowInspector, - showInspector, roomId, }) { usePreventScroll(); const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0); const { audioOutput } = useMediaHandler(); + const [showInspector] = useShowInspector(); + + const audioContext = useRef(); + if (!audioContext.current) audioContext.current = new AudioContext(); const { modalState: feedbackModalState, modalProps: feedbackModalProps } = useModalTriggerState(); @@ -151,6 +154,7 @@ export function InCallView({ getAvatar={renderAvatar} showName={items.length > 2 || item.focused} audioOutputDevice={audioOutput} + audioContext={audioContext.current} disableSpeakingIndicator={items.length < 3} {...rest} /> @@ -169,8 +173,6 @@ export function InCallView({ diff --git a/src/room/OverflowMenu.jsx b/src/room/OverflowMenu.jsx index 281995c..c5810f0 100644 --- a/src/room/OverflowMenu.jsx +++ b/src/room/OverflowMenu.jsx @@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal"; export function OverflowMenu({ roomId, - setShowInspector, - showInspector, inCall, groupCall, showInvite, @@ -88,13 +86,7 @@ export function OverflowMenu({ )} - {settingsModalState.isOpen && ( - - )} + {settingsModalState.isOpen && } {inviteModalState.isOpen && ( )} diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx index 4b395f8..795c249 100644 --- a/src/room/PTTCallView.tsx +++ b/src/room/PTTCallView.tsx @@ -84,8 +84,6 @@ interface Props { participants: RoomMember[]; userMediaFeeds: CallFeed[]; onLeave: () => void; - setShowInspector: (boolean) => void; - showInspector: boolean; } export const PTTCallView: React.FC = ({ @@ -97,8 +95,6 @@ export const PTTCallView: React.FC = ({ participants, userMediaFeeds, onLeave, - setShowInspector, - showInspector, }) => { const { modalState: inviteModalState, modalProps: inviteModalProps } = useModalTriggerState(); @@ -189,8 +185,6 @@ export const PTTCallView: React.FC = ({ { const { audioInput, audioInputs, @@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) { audioOutputs, setAudioOutput, } = useMediaHandler(); + const [spatialAudio, setSpatialAudio] = useSpatialAudio(); + const [showInspector, setShowInspector] = useShowInspector(); const downloadDebugLog = useDownloadDebugLog(); @@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) { isDismissable mobileFullScreen className={styles.settingsModal} - {...rest} + {...props} > )} + + setSpatialAudio(e.target.checked)} + /> + ); -} +}; diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts new file mode 100644 index 0000000..b0db79c --- /dev/null +++ b/src/settings/useSetting.ts @@ -0,0 +1,56 @@ +/* +Copyright 2022 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventEmitter } from "events"; +import { useMemo, useState, useEffect, useCallback } from "react"; + +// Bus to notify other useSetting consumers when a setting is changed +const settingsBus = new EventEmitter(); + +// Like useState, but reads from and persists the value to localStorage +const useSetting = ( + name: string, + defaultValue: T +): [T, (value: T) => void] => { + const key = useMemo(() => `matrix-setting-${name}`, [name]); + + const [value, setValue] = useState(() => { + const item = localStorage.getItem(key); + return item == null ? defaultValue : JSON.parse(item); + }); + + useEffect(() => { + settingsBus.on(name, setValue); + return () => { + settingsBus.off(name, setValue); + }; + }, [name, setValue]); + + return [ + value, + useCallback( + (newValue: T) => { + setValue(newValue); + localStorage.setItem(key, JSON.stringify(newValue)); + settingsBus.emit(name, newValue); + }, + [name, key, setValue] + ), + ]; +}; + +export const useSpatialAudio = () => useSetting("spatial-audio", false); +export const useShowInspector = () => useSetting("show-inspector", false); diff --git a/src/video-grid/VideoTile.jsx b/src/video-grid/VideoTile.jsx index 90780f7..2dd4192 100644 --- a/src/video-grid/VideoTile.jsx +++ b/src/video-grid/VideoTile.jsx @@ -14,57 +14,63 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React from "react"; +import React, { forwardRef } from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import styles from "./VideoTile.module.css"; import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg"; -export function VideoTile({ - className, - isLocal, - speaking, - audioMuted, - noVideo, - videoMuted, - screenshare, - avatar, - name, - showName, - mediaRef, - ...rest -}) { - return ( - - {(videoMuted || noVideo) && ( - <> -
- {avatar} - - )} - {screenshare ? ( -
- {`${name} is presenting`} -
- ) : ( - (showName || audioMuted || (videoMuted && !noVideo)) && ( -
- {audioMuted && !(videoMuted && !noVideo) && } - {videoMuted && !noVideo && } - {showName && {name}} +export const VideoTile = forwardRef( + ( + { + className, + isLocal, + speaking, + audioMuted, + noVideo, + videoMuted, + screenshare, + avatar, + name, + showName, + mediaRef, + ...rest + }, + ref + ) => { + return ( + + {(videoMuted || noVideo) && ( + <> +
+ {avatar} + + )} + {screenshare ? ( +
+ {`${name} is presenting`}
- ) - )} -