From c6b90803f8e8552733f670bbd5e04d7cf7fa8bfc Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Tue, 31 May 2022 10:43:05 -0400 Subject: [PATCH 1/2] Add spatial audio capabilities --- src/room/GroupCallView.jsx | 19 ----- src/room/InCallView.jsx | 12 ++-- src/room/LobbyView.jsx | 4 -- src/room/OverflowMenu.jsx | 10 +-- src/room/PTTCallView.tsx | 6 -- src/room/VideoPreview.jsx | 4 -- src/settings/SettingsModal.jsx | 18 ++++- src/settings/useSetting.ts | 56 +++++++++++++++ src/video-grid/VideoTile.jsx | 100 ++++++++++++++------------ src/video-grid/VideoTile.module.css | 4 ++ src/video-grid/VideoTileContainer.jsx | 11 ++- src/video-grid/useMediaStream.js | 60 ++++++++++++++++ 12 files changed, 205 insertions(+), 99 deletions(-) create mode 100644 src/settings/useSetting.ts diff --git a/src/room/GroupCallView.jsx b/src/room/GroupCallView.jsx index dcd9da3..2809b82 100644 --- a/src/room/GroupCallView.jsx +++ b/src/room/GroupCallView.jsx @@ -33,19 +33,6 @@ export function GroupCallView({ roomId, groupCall, }) { - const [showInspector, setShowInspector] = useState( - () => !!localStorage.getItem("matrix-group-call-inspector") - ); - const onChangeShowInspector = useCallback((show) => { - setShowInspector(show); - - if (show) { - localStorage.setItem("matrix-group-call-inspector", "true"); - } else { - localStorage.removeItem("matrix-group-call-inspector"); - } - }, []); - const { state, error, @@ -104,8 +91,6 @@ export function GroupCallView({ participants={participants} userMediaFeeds={userMediaFeeds} onLeave={onLeave} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} /> ); } else { @@ -126,8 +111,6 @@ export function GroupCallView({ isScreensharing={isScreensharing} localScreenshareFeed={localScreenshareFeed} screenshareFeeds={screenshareFeeds} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} roomId={roomId} /> ); @@ -156,8 +139,6 @@ export function GroupCallView({ localVideoMuted={localVideoMuted} toggleLocalVideoMuted={toggleLocalVideoMuted} toggleMicrophoneMuted={toggleMicrophoneMuted} - setShowInspector={onChangeShowInspector} - showInspector={showInspector} roomId={roomId} /> ); diff --git a/src/room/InCallView.jsx b/src/room/InCallView.jsx index a474963..2c6240e 100644 --- a/src/room/InCallView.jsx +++ b/src/room/InCallView.jsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { useCallback, useMemo } from "react"; +import React, { useCallback, useMemo, useRef } from "react"; import styles from "./InCallView.module.css"; import { HangupButton, @@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { usePreventScroll } from "@react-aria/overlays"; import { useMediaHandler } from "../settings/useMediaHandler"; +import { useShowInspector } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; const canScreenshare = "getDisplayMedia" in navigator.mediaDevices; @@ -57,14 +58,16 @@ export function InCallView({ toggleScreensharing, isScreensharing, screenshareFeeds, - setShowInspector, - showInspector, roomId, }) { usePreventScroll(); const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0); const { audioOutput } = useMediaHandler(); + const [showInspector] = useShowInspector(); + + const audioContext = useRef(); + if (!audioContext.current) audioContext.current = new AudioContext(); const { modalState: feedbackModalState, modalProps: feedbackModalProps } = useModalTriggerState(); @@ -151,6 +154,7 @@ export function InCallView({ getAvatar={renderAvatar} showName={items.length > 2 || item.focused} audioOutputDevice={audioOutput} + audioContext={audioContext.current} disableSpeakingIndicator={items.length < 3} {...rest} /> @@ -169,8 +173,6 @@ export function InCallView({ diff --git a/src/room/OverflowMenu.jsx b/src/room/OverflowMenu.jsx index 281995c..c5810f0 100644 --- a/src/room/OverflowMenu.jsx +++ b/src/room/OverflowMenu.jsx @@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal"; export function OverflowMenu({ roomId, - setShowInspector, - showInspector, inCall, groupCall, showInvite, @@ -88,13 +86,7 @@ export function OverflowMenu({ )} - {settingsModalState.isOpen && ( - - )} + {settingsModalState.isOpen && } {inviteModalState.isOpen && ( )} diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx index 4b395f8..795c249 100644 --- a/src/room/PTTCallView.tsx +++ b/src/room/PTTCallView.tsx @@ -84,8 +84,6 @@ interface Props { participants: RoomMember[]; userMediaFeeds: CallFeed[]; onLeave: () => void; - setShowInspector: (boolean) => void; - showInspector: boolean; } export const PTTCallView: React.FC = ({ @@ -97,8 +95,6 @@ export const PTTCallView: React.FC = ({ participants, userMediaFeeds, onLeave, - setShowInspector, - showInspector, }) => { const { modalState: inviteModalState, modalProps: inviteModalProps } = useModalTriggerState(); @@ -189,8 +185,6 @@ export const PTTCallView: React.FC = ({ { const { audioInput, audioInputs, @@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) { audioOutputs, setAudioOutput, } = useMediaHandler(); + const [spatialAudio, setSpatialAudio] = useSpatialAudio(); + const [showInspector, setShowInspector] = useShowInspector(); const downloadDebugLog = useDownloadDebugLog(); @@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) { isDismissable mobileFullScreen className={styles.settingsModal} - {...rest} + {...props} > )} + + setSpatialAudio(e.target.checked)} + /> + ); -} +}; diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts new file mode 100644 index 0000000..b0db79c --- /dev/null +++ b/src/settings/useSetting.ts @@ -0,0 +1,56 @@ +/* +Copyright 2022 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventEmitter } from "events"; +import { useMemo, useState, useEffect, useCallback } from "react"; + +// Bus to notify other useSetting consumers when a setting is changed +const settingsBus = new EventEmitter(); + +// Like useState, but reads from and persists the value to localStorage +const useSetting = ( + name: string, + defaultValue: T +): [T, (value: T) => void] => { + const key = useMemo(() => `matrix-setting-${name}`, [name]); + + const [value, setValue] = useState(() => { + const item = localStorage.getItem(key); + return item == null ? defaultValue : JSON.parse(item); + }); + + useEffect(() => { + settingsBus.on(name, setValue); + return () => { + settingsBus.off(name, setValue); + }; + }, [name, setValue]); + + return [ + value, + useCallback( + (newValue: T) => { + setValue(newValue); + localStorage.setItem(key, JSON.stringify(newValue)); + settingsBus.emit(name, newValue); + }, + [name, key, setValue] + ), + ]; +}; + +export const useSpatialAudio = () => useSetting("spatial-audio", false); +export const useShowInspector = () => useSetting("show-inspector", false); diff --git a/src/video-grid/VideoTile.jsx b/src/video-grid/VideoTile.jsx index 90780f7..2dd4192 100644 --- a/src/video-grid/VideoTile.jsx +++ b/src/video-grid/VideoTile.jsx @@ -14,57 +14,63 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React from "react"; +import React, { forwardRef } from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import styles from "./VideoTile.module.css"; import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg"; -export function VideoTile({ - className, - isLocal, - speaking, - audioMuted, - noVideo, - videoMuted, - screenshare, - avatar, - name, - showName, - mediaRef, - ...rest -}) { - return ( - - {(videoMuted || noVideo) && ( - <> -
- {avatar} - - )} - {screenshare ? ( -
- {`${name} is presenting`} -
- ) : ( - (showName || audioMuted || (videoMuted && !noVideo)) && ( -
- {audioMuted && !(videoMuted && !noVideo) && } - {videoMuted && !noVideo && } - {showName && {name}} +export const VideoTile = forwardRef( + ( + { + className, + isLocal, + speaking, + audioMuted, + noVideo, + videoMuted, + screenshare, + avatar, + name, + showName, + mediaRef, + ...rest + }, + ref + ) => { + return ( + + {(videoMuted || noVideo) && ( + <> +
+ {avatar} + + )} + {screenshare ? ( +
+ {`${name} is presenting`}
- ) - )} -