Merge remote-tracking branch 'origin/main' into dbkr/avoid-browser-index-import

src/index.css

@@ -26,6 +26,8 @@ limitations under the License.
   --inter-unicode-range: U+0000-20e2, U+20e4-23ce, U+23d0-24c1, U+24c3-259f,
     U+25c2-2664, U+2666-2763, U+2765-2b05, U+2b07-2b1b, U+2b1d-10FFFF;
   --primaryColor: #0dbd8b;
+  --primaryColor-20: #0dbd8b33;
+  --alert-20: #ff5b5533;
   --bgColor1: #15191e;
   --bgColor2: #21262c;
   --bgColor3: #444;

src/room/GroupCallView.jsx

@@ -33,19 +33,6 @@ export function GroupCallView({
   roomId,
   groupCall,
 }) {
-  const [showInspector, setShowInspector] = useState(
-    () => !!localStorage.getItem("matrix-group-call-inspector")
-  );
-  const onChangeShowInspector = useCallback((show) => {
-    setShowInspector(show);
-
-    if (show) {
-      localStorage.setItem("matrix-group-call-inspector", "true");
-    } else {
-      localStorage.removeItem("matrix-group-call-inspector");
-    }
-  }, []);
-
   const {
     state,
     error,
@@ -104,8 +91,6 @@ export function GroupCallView({
           participants={participants}
           userMediaFeeds={userMediaFeeds}
           onLeave={onLeave}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
         />
       );
     } else {
@@ -126,8 +111,6 @@ export function GroupCallView({
           isScreensharing={isScreensharing}
           localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
           roomId={roomId}
         />
       );
@@ -156,8 +139,6 @@ export function GroupCallView({
         localVideoMuted={localVideoMuted}
         toggleLocalVideoMuted={toggleLocalVideoMuted}
         toggleMicrophoneMuted={toggleMicrophoneMuted}
-        setShowInspector={onChangeShowInspector}
-        showInspector={showInspector}
         roomId={roomId}
       />
     );

src/room/InCallView.jsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useCallback, useMemo } from "react";
+import React, { useCallback, useMemo, useRef } from "react";
 import styles from "./InCallView.module.css";
 import {
   HangupButton,
@@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { usePreventScroll } from "@react-aria/overlays";
 import { useMediaHandler } from "../settings/useMediaHandler";
+import { useShowInspector } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
 
 const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@@ -57,14 +58,16 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   usePreventScroll();
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
 
   const { audioOutput } = useMediaHandler();
+  const [showInspector] = useShowInspector();
+
+  const audioContext = useRef();
+  if (!audioContext.current) audioContext.current = new AudioContext();
 
   const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
     useModalTriggerState();
@@ -151,6 +154,7 @@ export function InCallView({
               getAvatar={renderAvatar}
               showName={items.length > 2 || item.focused}
               audioOutputDevice={audioOutput}
+              audioContext={audioContext.current}
               disableSpeakingIndicator={items.length < 3}
               {...rest}
             />
@@ -169,8 +173,6 @@ export function InCallView({
         <OverflowMenu
           inCall
           roomId={roomId}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
           client={client}
           groupCall={groupCall}
           showInvite={true}

src/room/LobbyView.jsx

@@ -41,8 +41,6 @@ export function LobbyView({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   const { stream } = useCallFeed(localCallFeed);
@@ -101,8 +99,6 @@ export function LobbyView({
               localVideoMuted={localVideoMuted}
               toggleLocalVideoMuted={toggleLocalVideoMuted}
               toggleMicrophoneMuted={toggleMicrophoneMuted}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               stream={stream}
               audioOutput={audioOutput}
             />

src/room/OverflowMenu.jsx

@@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
 
 export function OverflowMenu({
   roomId,
-  setShowInspector,
-  showInspector,
   inCall,
   groupCall,
   showInvite,
@@ -88,13 +86,7 @@ export function OverflowMenu({
           </Menu>
         )}
       </PopoverMenuTrigger>
-      {settingsModalState.isOpen && (
+      {settingsModalState.isOpen && <SettingsModal {...settingsModalProps} />}
-        <SettingsModal
-          {...settingsModalProps}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
-        />
-      )}
       {inviteModalState.isOpen && (
         <InviteModal roomId={roomId} {...inviteModalProps} />
       )}

src/room/PTTButton.module.css

@@ -9,17 +9,15 @@
   background-color: #21262c;
   position: relative;
   padding: 0;
+  cursor: pointer;
 }
 
 .talking {
   background-color: #0dbd8b;
-  box-shadow: 0px 0px 0px 17px rgba(13, 189, 139, 0.2),
+  cursor: unset;
-    0px 0px 0px 34px rgba(13, 189, 139, 0.2);
 }
 
 .error {
   background-color: #ff5b55;
   border-color: #ff5b55;
-  box-shadow: 0px 0px 0px 17px rgba(255, 91, 85, 0.2),
-    0px 0px 0px 34px rgba(255, 91, 85, 0.2);
 }

src/room/PTTButton.tsx

@@ -16,6 +16,7 @@ limitations under the License.
 
 import React, { useCallback, useEffect, useState, createRef } from "react";
 import classNames from "classnames";
+import { useSpring, animated } from "@react-spring/web";
 
 import styles from "./PTTButton.module.css";
 import { ReactComponent as MicIcon } from "../icons/Mic.svg";
@@ -27,6 +28,7 @@ interface Props {
   activeSpeakerDisplayName: string;
   activeSpeakerAvatarUrl: string;
   activeSpeakerIsLocalUser: boolean;
+  activeSpeakerVolume: number;
   size: number;
   startTalking: () => void;
   stopTalking: () => void;
@@ -44,6 +46,7 @@ export const PTTButton: React.FC<Props> = ({
   activeSpeakerDisplayName,
   activeSpeakerAvatarUrl,
   activeSpeakerIsLocalUser,
+  activeSpeakerVolume,
   size,
   startTalking,
   stopTalking,
@@ -130,12 +133,32 @@ export const PTTButton: React.FC<Props> = ({
       );
     };
   }, [onWindowMouseUp, onWindowTouchEnd, onButtonTouchStart, buttonRef]);
+
+  const { shadow } = useSpring({
+    shadow: (Math.max(activeSpeakerVolume, -70) + 70) * 0.6,
+    config: {
+      clamp: true,
+      tension: 300,
+    },
+  });
+  const shadowColor = showTalkOverError
+    ? "var(--alert-20)"
+    : "var(--primaryColor-20)";
+
   return (
-    <button
+    <animated.button
       className={classNames(styles.pttButton, {
         [styles.talking]: activeSpeakerUserId,
         [styles.error]: showTalkOverError,
       })}
+      style={{
+        boxShadow: shadow.to(
+          (s) =>
+            `0px 0px 0px ${s}px ${shadowColor}, 0px 0px 0px ${
+              2 * s
+            }px ${shadowColor}`
+        ),
+      }}
       onMouseDown={onButtonMouseDown}
       ref={buttonRef}
     >
@@ -154,6 +177,6 @@ export const PTTButton: React.FC<Props> = ({
           className={styles.avatar}
         />
       )}
-    </button>
+    </animated.button>
   );
 };

src/room/PTTCallView.tsx

@@ -44,8 +44,11 @@ function getPromptText(
   activeSpeakerIsLocalUser: boolean,
   talkOverEnabled: boolean,
   activeSpeakerUserId: string,
-  activeSpeakerDisplayName: string
+  activeSpeakerDisplayName: string,
+  connected: boolean
 ): string {
+  if (!connected) return "Connection Lost";
+
   const isTouchScreen = Boolean(window.ontouchstart !== undefined);
 
   if (showTalkOverError) {
@@ -84,8 +87,6 @@ interface Props {
   participants: RoomMember[];
   userMediaFeeds: CallFeed[];
   onLeave: () => void;
-  setShowInspector: (boolean) => void;
-  showInspector: boolean;
 }
 
 export const PTTCallView: React.FC<Props> = ({
@@ -97,8 +98,6 @@ export const PTTCallView: React.FC<Props> = ({
   participants,
   userMediaFeeds,
   onLeave,
-  setShowInspector,
-  showInspector,
 }) => {
   const { modalState: inviteModalState, modalProps: inviteModalProps } =
     useModalTriggerState();
@@ -124,9 +123,11 @@ export const PTTCallView: React.FC<Props> = ({
     talkOverEnabled,
     setTalkOverEnabled,
     activeSpeakerUserId,
+    activeSpeakerVolume,
     startTalking,
     stopTalking,
     transmitBlocked,
+    connected,
   } = usePTT(
     client,
     groupCall,
@@ -189,8 +190,6 @@ export const PTTCallView: React.FC<Props> = ({
           <OverflowMenu
             inCall
             roomId={roomId}
-            setShowInspector={setShowInspector}
-            showInspector={showInspector}
             client={client}
             groupCall={groupCall}
             showInvite={false}
@@ -223,6 +222,7 @@ export const PTTCallView: React.FC<Props> = ({
             activeSpeakerDisplayName={activeSpeakerDisplayName}
             activeSpeakerAvatarUrl={activeSpeakerAvatarUrl}
             activeSpeakerIsLocalUser={activeSpeakerIsLocalUser}
+            activeSpeakerVolume={activeSpeakerVolume}
             size={pttButtonSize}
             startTalking={startTalking}
             stopTalking={stopTalking}
@@ -234,7 +234,8 @@ export const PTTCallView: React.FC<Props> = ({
               activeSpeakerIsLocalUser,
               talkOverEnabled,
               activeSpeakerUserId,
-              activeSpeakerDisplayName
+              activeSpeakerDisplayName,
+              connected
             )}
           </p>
           {userMediaFeeds.map((callFeed) => (

src/room/VideoPreview.jsx

@@ -35,8 +35,6 @@ export function VideoPreview({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   audioOutput,
   stream,
 }) {
@@ -83,8 +81,6 @@ export function VideoPreview({
             />
             <OverflowMenu
               roomId={roomId}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               client={client}
               feedbackModalState={feedbackModalState}
               feedbackModalProps={feedbackModalProps}

src/room/useGroupCall.ts

@@ -18,10 +18,57 @@ import { useCallback, useEffect, useState } from "react";
 import {
   GroupCallEvent,
   GroupCallState,
+  GroupCall,
 } from "matrix-js-sdk/src/webrtc/groupCall";
+import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+import { RoomMember } from "matrix-js-sdk/src/models/room-member";
+
 import { usePageUnload } from "./usePageUnload";
 
-export function useGroupCall(groupCall) {
+export interface UseGroupCallType {
+  state: GroupCallState;
+  calls: MatrixCall[];
+  localCallFeed: CallFeed;
+  activeSpeaker: string;
+  userMediaFeeds: CallFeed[];
+  microphoneMuted: boolean;
+  localVideoMuted: boolean;
+  error: Error;
+  initLocalCallFeed: () => void;
+  enter: () => void;
+  leave: () => void;
+  toggleLocalVideoMuted: () => void;
+  toggleMicrophoneMuted: () => void;
+  toggleScreensharing: () => void;
+  requestingScreenshare: boolean;
+  isScreensharing: boolean;
+  screenshareFeeds: CallFeed[];
+  localScreenshareFeed: CallFeed;
+  localDesktopCapturerSourceId: string;
+  participants: RoomMember[];
+  hasLocalParticipant: boolean;
+}
+
+interface State {
+  state: GroupCallState;
+  calls: MatrixCall[];
+  localCallFeed: CallFeed;
+  activeSpeaker: string;
+  userMediaFeeds: CallFeed[];
+  error: Error;
+  microphoneMuted: boolean;
+  localVideoMuted: boolean;
+  screenshareFeeds: CallFeed[];
+  localScreenshareFeed: CallFeed;
+  localDesktopCapturerSourceId: string;
+  isScreensharing: boolean;
+  requestingScreenshare: boolean;
+  participants: RoomMember[];
+  hasLocalParticipant: boolean;
+}
+
+export function useGroupCall(groupCall: GroupCall): UseGroupCallType {
   const [
     {
       state,
@@ -41,20 +88,25 @@ export function useGroupCall(groupCall) {
       requestingScreenshare,
     },
     setState,
-  ] = useState({
+  ] = useState<State>({
     state: GroupCallState.LocalCallFeedUninitialized,
     calls: [],
+    localCallFeed: null,
+    activeSpeaker: null,
     userMediaFeeds: [],
+    error: null,
     microphoneMuted: false,
     localVideoMuted: false,
-    screenshareFeeds: [],
     isScreensharing: false,
+    screenshareFeeds: [],
+    localScreenshareFeed: null,
+    localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
     hasLocalParticipant: false,
   });
 
-  const updateState = (state) =>
+  const updateState = (state: Partial<State>) =>
     setState((prevState) => ({ ...prevState, ...state }));
 
   useEffect(() => {
@@ -75,25 +127,28 @@ export function useGroupCall(groupCall) {
       });
     }
 
-    function onUserMediaFeedsChanged(userMediaFeeds) {
+    function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void {
       updateState({
         userMediaFeeds: [...userMediaFeeds],
       });
     }
 
-    function onScreenshareFeedsChanged(screenshareFeeds) {
+    function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void {
       updateState({
         screenshareFeeds: [...screenshareFeeds],
       });
     }
 
-    function onActiveSpeakerChanged(activeSpeaker) {
+    function onActiveSpeakerChanged(activeSpeaker: string): void {
       updateState({
         activeSpeaker: activeSpeaker,
       });
     }
 
-    function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) {
+    function onLocalMuteStateChanged(
+      microphoneMuted: boolean,
+      localVideoMuted: boolean
+    ): void {
       updateState({
         microphoneMuted,
         localVideoMuted,
@@ -101,10 +156,10 @@ export function useGroupCall(groupCall) {
     }
 
     function onLocalScreenshareStateChanged(
-      isScreensharing,
+      isScreensharing: boolean,
-      localScreenshareFeed,
+      localScreenshareFeed: CallFeed,
-      localDesktopCapturerSourceId
+      localDesktopCapturerSourceId: string
-    ) {
+    ): void {
       updateState({
         isScreensharing,
         localScreenshareFeed,
@@ -112,13 +167,13 @@ export function useGroupCall(groupCall) {
       });
     }
 
-    function onCallsChanged(calls) {
+    function onCallsChanged(calls: MatrixCall[]): void {
       updateState({
         calls: [...calls],
       });
     }
 
-    function onParticipantsChanged(participants) {
+    function onParticipantsChanged(participants: RoomMember[]): void {
       updateState({
         participants: [...participants],
         hasLocalParticipant: groupCall.hasLocalParticipant(),

src/room/usePTT.ts

@@ -15,10 +15,11 @@ limitations under the License.
 */
 
 import { useCallback, useEffect, useState } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
 import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
 import { logger } from "matrix-js-sdk/src/logger";
+import { SyncState } from "matrix-js-sdk/src/sync";
 
 import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
 
@@ -30,6 +31,21 @@ function getActiveSpeakerFeed(
 ): CallFeed | null {
   const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
 
+  // make sure the feeds are in a deterministic order so every client picks
+  // the same one as the active speaker. The custom sort function sorts
+  // by user ID, so needs a collator of some kind to compare. We make a
+  // specific one to help ensure every client sorts the same way
+  // although of course user IDs shouldn't contain accented characters etc.
+  // anyway).
+  const collator = new Intl.Collator("en", {
+    sensitivity: "variant",
+    usage: "sort",
+    ignorePunctuation: false,
+  });
+  activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number =>
+    collator.compare(a.userId, b.userId)
+  );
+
   let activeSpeakerFeed = null;
   let highestPowerLevel = null;
   for (const feed of activeSpeakerFeeds) {
@@ -49,9 +65,15 @@ export interface PTTState {
   talkOverEnabled: boolean;
   setTalkOverEnabled: (boolean) => void;
   activeSpeakerUserId: string;
+  activeSpeakerVolume: number;
   startTalking: () => void;
   stopTalking: () => void;
   transmitBlocked: boolean;
+  // connected is actually an indication of whether we're connected to the HS
+  // (ie. the client's syncing state) rather than media connection, since
+  // it's peer to peer so we can't really say which peer is 'disconnected' if
+  // there's only one other person in the call and they've lost Internet.
+  connected: boolean;
 }
 
 export const usePTT = (
@@ -87,6 +109,7 @@ export const usePTT = (
       isAdmin,
       talkOverEnabled,
       activeSpeakerUserId,
+      activeSpeakerVolume,
       transmitBlocked,
     },
     setState,
@@ -100,6 +123,7 @@ export const usePTT = (
       talkOverEnabled: false,
       pttButtonHeld: false,
       activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
+      activeSpeakerVolume: -Infinity,
       transmitBlocked: false,
     };
   });
@@ -131,15 +155,11 @@ export const usePTT = (
       playClip(PTTClipID.BLOCKED);
     }
 
-    setState((prevState) => {
+    setState((prevState) => ({
-      return {
+      ...prevState,
-        ...prevState,
+      activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
-        activeSpeakerUserId: activeSpeakerFeed
+      transmitBlocked: blocked,
-          ? activeSpeakerFeed.userId
+    }));
-          : null,
-        transmitBlocked: blocked,
-      };
-    });
   }, [
     playClip,
     groupCall,
@@ -152,7 +172,7 @@ export const usePTT = (
 
   useEffect(() => {
     for (const callFeed of userMediaFeeds) {
-      callFeed.addListener(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
+      callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
     }
 
     const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
@@ -164,14 +184,30 @@ export const usePTT = (
 
     return () => {
       for (const callFeed of userMediaFeeds) {
-        callFeed.removeListener(
+        callFeed.off(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
-          CallFeedEvent.MuteStateChanged,
-          onMuteStateChanged
-        );
       }
     };
   }, [userMediaFeeds, onMuteStateChanged, groupCall]);
 
+  const onVolumeChanged = useCallback((volume: number) => {
+    setState((prevState) => ({
+      ...prevState,
+      activeSpeakerVolume: volume,
+    }));
+  }, []);
+
+  useEffect(() => {
+    const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
+    activeSpeakerFeed?.on(CallFeedEvent.VolumeChanged, onVolumeChanged);
+    return () => {
+      activeSpeakerFeed?.off(CallFeedEvent.VolumeChanged, onVolumeChanged);
+      setState((prevState) => ({
+        ...prevState,
+        activeSpeakerVolume: -Infinity,
+      }));
+    };
+  }, [activeSpeakerUserId, onVolumeChanged, userMediaFeeds, groupCall]);
+
   const startTalking = useCallback(async () => {
     if (pttButtonHeld) return;
 
@@ -211,6 +247,17 @@ export const usePTT = (
     setMicMuteWrapper(true);
   }, [setMicMuteWrapper]);
 
+  // separate state for connected: we set it separately from other things
+  // in the client sync callback
+  const [connected, setConnected] = useState(true);
+
+  const onClientSync = useCallback(
+    (syncState: SyncState) => {
+      setConnected(syncState !== SyncState.Error);
+    },
+    [setConnected]
+  );
+
   useEffect(() => {
     function onKeyDown(event: KeyboardEvent): void {
       if (event.code === "Space") {
@@ -260,8 +307,18 @@ export const usePTT = (
     pttButtonHeld,
     enablePTTButton,
     setMicMuteWrapper,
+    client,
+    onClientSync,
   ]);
 
+  useEffect(() => {
+    client.on(ClientEvent.Sync, onClientSync);
+
+    return () => {
+      client.removeListener(ClientEvent.Sync, onClientSync);
+    };
+  }, [client, onClientSync]);
+
   const setTalkOverEnabled = useCallback((talkOverEnabled) => {
     setState((prevState) => ({
       ...prevState,
@@ -275,8 +332,10 @@ export const usePTT = (
     talkOverEnabled,
     setTalkOverEnabled,
     activeSpeakerUserId,
+    activeSpeakerVolume,
     startTalking,
     stopTalking,
     transmitBlocked,
+    connected,
   };
 };

src/settings/SettingsModal.jsx

@@ -24,12 +24,13 @@ import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
 import { SelectInput } from "../input/SelectInput";
 import { Item } from "@react-stately/collections";
 import { useMediaHandler } from "./useMediaHandler";
+import { useSpatialAudio, useShowInspector } from "./useSetting";
 import { FieldRow, InputField } from "../input/Input";
 import { Button } from "../button";
 import { useDownloadDebugLog } from "./submit-rageshake";
 import { Body } from "../typography/Typography";
 
-export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
+export const SettingsModal = (props) => {
   const {
     audioInput,
     audioInputs,
@@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
     audioOutputs,
     setAudioOutput,
   } = useMediaHandler();
+  const [spatialAudio, setSpatialAudio] = useSpatialAudio();
+  const [showInspector, setShowInspector] = useShowInspector();
 
   const downloadDebugLog = useDownloadDebugLog();
 
@@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       isDismissable
       mobileFullScreen
       className={styles.settingsModal}
-      {...rest}
+      {...props}
     >
       <TabContainer className={styles.tabContainer}>
         <TabItem
@@ -81,6 +84,15 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
               ))}
             </SelectInput>
           )}
+          <FieldRow>
+            <InputField
+              id="spatialAudio"
+              label="Spatial audio (experimental)"
+              type="checkbox"
+              checked={spatialAudio}
+              onChange={(e) => setSpatialAudio(e.target.checked)}
+            />
+          </FieldRow>
         </TabItem>
         <TabItem
           title={
@@ -130,4 +142,4 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       </TabContainer>
     </Modal>
   );
-}
+};

src/settings/useSetting.ts

@@ -0,0 +1,56 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { useMemo, useState, useEffect, useCallback } from "react";
+
+// Bus to notify other useSetting consumers when a setting is changed
+const settingsBus = new EventEmitter();
+
+// Like useState, but reads from and persists the value to localStorage
+const useSetting = <T>(
+  name: string,
+  defaultValue: T
+): [T, (value: T) => void] => {
+  const key = useMemo(() => `matrix-setting-${name}`, [name]);
+
+  const [value, setValue] = useState<T>(() => {
+    const item = localStorage.getItem(key);
+    return item == null ? defaultValue : JSON.parse(item);
+  });
+
+  useEffect(() => {
+    settingsBus.on(name, setValue);
+    return () => {
+      settingsBus.off(name, setValue);
+    };
+  }, [name, setValue]);
+
+  return [
+    value,
+    useCallback(
+      (newValue: T) => {
+        setValue(newValue);
+        localStorage.setItem(key, JSON.stringify(newValue));
+        settingsBus.emit(name, newValue);
+      },
+      [name, key, setValue]
+    ),
+  ];
+};
+
+export const useSpatialAudio = () => useSetting("spatial-audio", false);
+export const useShowInspector = () => useSetting("show-inspector", false);

src/video-grid/VideoTile.jsx

@@ -14,57 +14,63 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, { forwardRef } from "react";
 import { animated } from "@react-spring/web";
 import classNames from "classnames";
 import styles from "./VideoTile.module.css";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
 
-export function VideoTile({
+export const VideoTile = forwardRef(
-  className,
+  (
-  isLocal,
+    {
-  speaking,
+      className,
-  audioMuted,
+      isLocal,
-  noVideo,
+      speaking,
-  videoMuted,
+      audioMuted,
-  screenshare,
+      noVideo,
-  avatar,
+      videoMuted,
-  name,
+      screenshare,
-  showName,
+      avatar,
-  mediaRef,
+      name,
-  ...rest
+      showName,
-}) {
+      mediaRef,
-  return (
+      ...rest
-    <animated.div
+    },
-      className={classNames(styles.videoTile, className, {
+    ref
-        [styles.isLocal]: isLocal,
+  ) => {
-        [styles.speaking]: speaking,
+    return (
-        [styles.muted]: audioMuted,
+      <animated.div
-        [styles.screenshare]: screenshare,
+        className={classNames(styles.videoTile, className, {
-      })}
+          [styles.isLocal]: isLocal,
-      {...rest}
+          [styles.speaking]: speaking,
-    >
+          [styles.muted]: audioMuted,
-      {(videoMuted || noVideo) && (
+          [styles.screenshare]: screenshare,
-        <>
+        })}
-          <div className={styles.videoMutedOverlay} />
+        ref={ref}
-          {avatar}
+        {...rest}
-        </>
+      >
-      )}
+        {(videoMuted || noVideo) && (
-      {screenshare ? (
+          <>
-        <div className={styles.presenterLabel}>
+            <div className={styles.videoMutedOverlay} />
-          <span>{`${name} is presenting`}</span>
+            {avatar}
-        </div>
+          </>
-      ) : (
+        )}
-        (showName || audioMuted || (videoMuted && !noVideo)) && (
+        {screenshare ? (
-          <div className={styles.memberName}>
+          <div className={styles.presenterLabel}>
-            {audioMuted && !(videoMuted && !noVideo) && <MicMutedIcon />}
+            <span>{`${name} is presenting`}</span>
-            {videoMuted && !noVideo && <VideoMutedIcon />}
-            {showName && <span title={name}>{name}</span>}
           </div>
-        )
+        ) : (
-      )}
+          (showName || audioMuted || (videoMuted && !noVideo)) && (
-      <video ref={mediaRef} playsInline disablePictureInPicture />
+            <div className={styles.memberName}>
-    </animated.div>
+              {audioMuted && !(videoMuted && !noVideo) && <MicMutedIcon />}
-  );
+              {videoMuted && !noVideo && <VideoMutedIcon />}
-}
+              {showName && <span title={name}>{name}</span>}
+            </div>
+          )
+        )}
+        <video ref={mediaRef} playsInline disablePictureInPicture />
+      </animated.div>
+    );
+  }
+);

src/video-grid/VideoTile.module.css

@@ -5,6 +5,10 @@
   overflow: hidden;
   cursor: pointer;
   touch-action: none;
+
+  /* HACK: This has no visual effect due to the short duration, but allows the
+  JS to detect movement via the transform property for audio spatialization */
+  transition: transform 0.000000001s;
 }
 
 .videoTile * {

src/video-grid/VideoTileContainer.jsx

@@ -17,7 +17,7 @@ limitations under the License.
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 import React from "react";
 import { useCallFeed } from "./useCallFeed";
-import { useMediaStream } from "./useMediaStream";
+import { useSpatialMediaStream } from "./useMediaStream";
 import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 
@@ -28,6 +28,7 @@ export function VideoTileContainer({
   getAvatar,
   showName,
   audioOutputDevice,
+  audioContext,
   disableSpeakingIndicator,
   ...rest
 }) {
@@ -42,7 +43,12 @@ export function VideoTileContainer({
     member,
   } = useCallFeed(item.callFeed);
   const { rawDisplayName } = useRoomMemberName(member);
-  const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
+  const [tileRef, mediaRef] = useSpatialMediaStream(
+    stream,
+    audioOutputDevice,
+    audioContext,
+    isLocal
+  );
 
   // Firefox doesn't respect the disablePictureInPicture attribute
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@@ -57,6 +63,7 @@ export function VideoTileContainer({
       screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
       name={rawDisplayName}
       showName={showName}
+      ref={tileRef}
       mediaRef={mediaRef}
       avatar={getAvatar && getAvatar(member, width, height)}
       {...rest}

src/video-grid/useMediaStream.js

@@ -16,6 +16,8 @@ limitations under the License.
 
 import { useRef, useEffect } from "react";
 
+import { useSpatialAudio } from "../settings/useSetting";
+
 export function useMediaStream(stream, audioOutputDevice, mute = false) {
   const mediaRef = useRef();
 
@@ -55,7 +57,8 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
       mediaRef.current !== undefined
     ) {
       console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
-      mediaRef.current.setSinkId(audioOutputDevice);
+      // Chrome for Android doesn't support this
+      mediaRef.current.setSinkId?.(audioOutputDevice);
     }
   }, [audioOutputDevice]);
 
@@ -73,3 +76,69 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
 
   return mediaRef;
 }
+
+export const useSpatialMediaStream = (
+  stream,
+  audioOutputDevice,
+  audioContext,
+  mute = false
+) => {
+  const tileRef = useRef();
+  const [spatialAudio] = useSpatialAudio();
+  // If spatial audio is enabled, we handle audio separately from the video element
+  const mediaRef = useMediaStream(
+    stream,
+    audioOutputDevice,
+    spatialAudio || mute
+  );
+
+  const pannerNodeRef = useRef();
+  if (!pannerNodeRef.current) {
+    pannerNodeRef.current = new PannerNode(audioContext, {
+      panningModel: "HRTF",
+      refDistance: 3,
+    });
+  }
+
+  const sourceRef = useRef();
+
+  useEffect(() => {
+    if (spatialAudio && tileRef.current && !mute) {
+      if (!sourceRef.current) {
+        sourceRef.current = audioContext.createMediaStreamSource(stream);
+      }
+
+      const tile = tileRef.current;
+      const source = sourceRef.current;
+      const pannerNode = pannerNodeRef.current;
+
+      const updatePosition = () => {
+        const bounds = tile.getBoundingClientRect();
+        const windowSize = Math.max(window.innerWidth, window.innerHeight);
+        // Position the source relative to its placement in the window
+        pannerNodeRef.current.positionX.value =
+          (bounds.x + bounds.width / 2) / windowSize - 0.5;
+        pannerNodeRef.current.positionY.value =
+          (bounds.y + bounds.height / 2) / windowSize - 0.5;
+        // Put the source in front of the listener
+        pannerNodeRef.current.positionZ.value = -2;
+      };
+
+      updatePosition();
+      source.connect(pannerNode);
+      pannerNode.connect(audioContext.destination);
+      // HACK: We abuse the CSS transitionrun event to detect when the tile
+      // moves, because useMeasure, IntersectionObserver, etc. all have no
+      // ability to track changes in the CSS transform property
+      tile.addEventListener("transitionrun", updatePosition);
+
+      return () => {
+        tile.removeEventListener("transitionrun", updatePosition);
+        source.disconnect();
+        pannerNode.disconnect();
+      };
+    }
+  }, [stream, spatialAudio, audioContext, mute]);
+
+  return [tileRef, mediaRef];
+};