Merge pull request #365 from robintown/spatial-audio

Spatial audio

src/room/GroupCallView.jsx

@@ -33,19 +33,6 @@ export function GroupCallView({
   roomId,
   groupCall,
 }) {
-  const [showInspector, setShowInspector] = useState(
-    () => !!localStorage.getItem("matrix-group-call-inspector")
-  );
-  const onChangeShowInspector = useCallback((show) => {
-    setShowInspector(show);
-
-    if (show) {
-      localStorage.setItem("matrix-group-call-inspector", "true");
-    } else {
-      localStorage.removeItem("matrix-group-call-inspector");
-    }
-  }, []);
-
   const {
     state,
     error,
@@ -104,8 +91,6 @@ export function GroupCallView({
           participants={participants}
           userMediaFeeds={userMediaFeeds}
           onLeave={onLeave}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
         />
       );
     } else {
@@ -126,8 +111,6 @@ export function GroupCallView({
           isScreensharing={isScreensharing}
           localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
           roomId={roomId}
         />
       );
@@ -156,8 +139,6 @@ export function GroupCallView({
         localVideoMuted={localVideoMuted}
         toggleLocalVideoMuted={toggleLocalVideoMuted}
         toggleMicrophoneMuted={toggleMicrophoneMuted}
-        setShowInspector={onChangeShowInspector}
-        showInspector={showInspector}
         roomId={roomId}
       />
     );

src/room/InCallView.jsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useCallback, useMemo } from "react";
+import React, { useCallback, useMemo, useRef } from "react";
 import styles from "./InCallView.module.css";
 import {
   HangupButton,
@@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { usePreventScroll } from "@react-aria/overlays";
 import { useMediaHandler } from "../settings/useMediaHandler";
+import { useShowInspector } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
 
 const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@@ -57,14 +58,16 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   usePreventScroll();
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
 
   const { audioOutput } = useMediaHandler();
+  const [showInspector] = useShowInspector();
+
+  const audioContext = useRef();
+  if (!audioContext.current) audioContext.current = new AudioContext();
 
   const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
     useModalTriggerState();
@@ -151,6 +154,7 @@ export function InCallView({
               getAvatar={renderAvatar}
               showName={items.length > 2 || item.focused}
               audioOutputDevice={audioOutput}
+              audioContext={audioContext.current}
               disableSpeakingIndicator={items.length < 3}
               {...rest}
             />
@@ -169,8 +173,6 @@ export function InCallView({
         <OverflowMenu
           inCall
           roomId={roomId}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
           client={client}
           groupCall={groupCall}
           showInvite={true}

src/room/LobbyView.jsx

@@ -41,8 +41,6 @@ export function LobbyView({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   const { stream } = useCallFeed(localCallFeed);
@@ -101,8 +99,6 @@ export function LobbyView({
               localVideoMuted={localVideoMuted}
               toggleLocalVideoMuted={toggleLocalVideoMuted}
               toggleMicrophoneMuted={toggleMicrophoneMuted}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               stream={stream}
               audioOutput={audioOutput}
             />

src/room/OverflowMenu.jsx

@@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
 
 export function OverflowMenu({
   roomId,
-  setShowInspector,
-  showInspector,
   inCall,
   groupCall,
   showInvite,
@@ -88,13 +86,7 @@ export function OverflowMenu({
           </Menu>
         )}
       </PopoverMenuTrigger>
-      {settingsModalState.isOpen && (
+      {settingsModalState.isOpen && <SettingsModal {...settingsModalProps} />}
-        <SettingsModal
-          {...settingsModalProps}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
-        />
-      )}
       {inviteModalState.isOpen && (
         <InviteModal roomId={roomId} {...inviteModalProps} />
       )}

src/room/PTTCallView.tsx

@@ -87,8 +87,6 @@ interface Props {
   participants: RoomMember[];
   userMediaFeeds: CallFeed[];
   onLeave: () => void;
-  setShowInspector: (boolean) => void;
-  showInspector: boolean;
 }
 
 export const PTTCallView: React.FC<Props> = ({
@@ -100,8 +98,6 @@ export const PTTCallView: React.FC<Props> = ({
   participants,
   userMediaFeeds,
   onLeave,
-  setShowInspector,
-  showInspector,
 }) => {
   const { modalState: inviteModalState, modalProps: inviteModalProps } =
     useModalTriggerState();
@@ -193,8 +189,6 @@ export const PTTCallView: React.FC<Props> = ({
           <OverflowMenu
             inCall
             roomId={roomId}
-            setShowInspector={setShowInspector}
-            showInspector={showInspector}
             client={client}
             groupCall={groupCall}
             showInvite={false}

src/room/VideoPreview.jsx

@@ -35,8 +35,6 @@ export function VideoPreview({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   audioOutput,
   stream,
 }) {
@@ -83,8 +81,6 @@ export function VideoPreview({
             />
             <OverflowMenu
               roomId={roomId}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               client={client}
               feedbackModalState={feedbackModalState}
               feedbackModalProps={feedbackModalProps}

src/settings/SettingsModal.jsx

@@ -24,12 +24,13 @@ import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
 import { SelectInput } from "../input/SelectInput";
 import { Item } from "@react-stately/collections";
 import { useMediaHandler } from "./useMediaHandler";
+import { useSpatialAudio, useShowInspector } from "./useSetting";
 import { FieldRow, InputField } from "../input/Input";
 import { Button } from "../button";
 import { useDownloadDebugLog } from "./submit-rageshake";
 import { Body } from "../typography/Typography";
 
-export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
+export const SettingsModal = (props) => {
   const {
     audioInput,
     audioInputs,
@@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
     audioOutputs,
     setAudioOutput,
   } = useMediaHandler();
+  const [spatialAudio, setSpatialAudio] = useSpatialAudio();
+  const [showInspector, setShowInspector] = useShowInspector();
 
   const downloadDebugLog = useDownloadDebugLog();
 
@@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       isDismissable
       mobileFullScreen
       className={styles.settingsModal}
-      {...rest}
+      {...props}
     >
       <TabContainer className={styles.tabContainer}>
         <TabItem
@@ -81,6 +84,15 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
               ))}
             </SelectInput>
           )}
+          <FieldRow>
+            <InputField
+              id="spatialAudio"
+              label="Spatial audio (experimental)"
+              type="checkbox"
+              checked={spatialAudio}
+              onChange={(e) => setSpatialAudio(e.target.checked)}
+            />
+          </FieldRow>
         </TabItem>
         <TabItem
           title={
@@ -130,4 +142,4 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       </TabContainer>
     </Modal>
   );
-}
+};

src/settings/useSetting.ts

@@ -0,0 +1,56 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { useMemo, useState, useEffect, useCallback } from "react";
+
+// Bus to notify other useSetting consumers when a setting is changed
+const settingsBus = new EventEmitter();
+
+// Like useState, but reads from and persists the value to localStorage
+const useSetting = <T>(
+  name: string,
+  defaultValue: T
+): [T, (value: T) => void] => {
+  const key = useMemo(() => `matrix-setting-${name}`, [name]);
+
+  const [value, setValue] = useState<T>(() => {
+    const item = localStorage.getItem(key);
+    return item == null ? defaultValue : JSON.parse(item);
+  });
+
+  useEffect(() => {
+    settingsBus.on(name, setValue);
+    return () => {
+      settingsBus.off(name, setValue);
+    };
+  }, [name, setValue]);
+
+  return [
+    value,
+    useCallback(
+      (newValue: T) => {
+        setValue(newValue);
+        localStorage.setItem(key, JSON.stringify(newValue));
+        settingsBus.emit(name, newValue);
+      },
+      [name, key, setValue]
+    ),
+  ];
+};
+
+export const useSpatialAudio = () => useSetting("spatial-audio", false);
+export const useShowInspector = () => useSetting("show-inspector", false);

src/video-grid/VideoTile.jsx

@@ -14,57 +14,63 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, { forwardRef } from "react";
 import { animated } from "@react-spring/web";
 import classNames from "classnames";
 import styles from "./VideoTile.module.css";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
 
-export function VideoTile({
+export const VideoTile = forwardRef(
-  className,
+  (
-  isLocal,
+    {
-  speaking,
+      className,
-  audioMuted,
+      isLocal,
-  noVideo,
+      speaking,
-  videoMuted,
+      audioMuted,
-  screenshare,
+      noVideo,
-  avatar,
+      videoMuted,
-  name,
+      screenshare,
-  showName,
+      avatar,
-  mediaRef,
+      name,
-  ...rest
+      showName,
-}) {
+      mediaRef,
-  return (
+      ...rest
-    <animated.div
+    },
-      className={classNames(styles.videoTile, className, {
+    ref
-        [styles.isLocal]: isLocal,
+  ) => {
-        [styles.speaking]: speaking,
+    return (
-        [styles.muted]: audioMuted,
+      <animated.div
-        [styles.screenshare]: screenshare,
+        className={classNames(styles.videoTile, className, {
-      })}
+          [styles.isLocal]: isLocal,
-      {...rest}
+          [styles.speaking]: speaking,
-    >
+          [styles.muted]: audioMuted,
-      {(videoMuted || noVideo) && (
+          [styles.screenshare]: screenshare,
-        <>
+        })}
-          <div className={styles.videoMutedOverlay} />
+        ref={ref}
-          {avatar}
+        {...rest}
-        </>
+      >
-      )}
+        {(videoMuted || noVideo) && (
-      {screenshare ? (
+          <>
-        <div className={styles.presenterLabel}>
+            <div className={styles.videoMutedOverlay} />
-          <span>{`${name} is presenting`}</span>
+            {avatar}
-        </div>
+          </>
-      ) : (
+        )}
-        (showName || audioMuted || (videoMuted && !noVideo)) && (
+        {screenshare ? (
-          <div className={styles.memberName}>
+          <div className={styles.presenterLabel}>
-            {audioMuted && !(videoMuted && !noVideo) && <MicMutedIcon />}
+            <span>{`${name} is presenting`}</span>
-            {videoMuted && !noVideo && <VideoMutedIcon />}
-            {showName && <span title={name}>{name}</span>}
           </div>
-        )
+        ) : (
-      )}
+          (showName || audioMuted || (videoMuted && !noVideo)) && (
-      <video ref={mediaRef} playsInline disablePictureInPicture />
+            <div className={styles.memberName}>
-    </animated.div>
+              {audioMuted && !(videoMuted && !noVideo) && <MicMutedIcon />}
-  );
+              {videoMuted && !noVideo && <VideoMutedIcon />}
-}
+              {showName && <span title={name}>{name}</span>}
+            </div>
+          )
+        )}
+        <video ref={mediaRef} playsInline disablePictureInPicture />
+      </animated.div>
+    );
+  }
+);

src/video-grid/VideoTile.module.css

@@ -5,6 +5,10 @@
   overflow: hidden;
   cursor: pointer;
   touch-action: none;
+
+  /* HACK: This has no visual effect due to the short duration, but allows the
+  JS to detect movement via the transform property for audio spatialization */
+  transition: transform 0.000000001s;
 }
 
 .videoTile * {

src/video-grid/VideoTileContainer.jsx

@@ -17,7 +17,7 @@ limitations under the License.
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 import React from "react";
 import { useCallFeed } from "./useCallFeed";
-import { useMediaStream } from "./useMediaStream";
+import { useSpatialMediaStream } from "./useMediaStream";
 import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 
@@ -28,6 +28,7 @@ export function VideoTileContainer({
   getAvatar,
   showName,
   audioOutputDevice,
+  audioContext,
   disableSpeakingIndicator,
   ...rest
 }) {
@@ -42,7 +43,12 @@ export function VideoTileContainer({
     member,
   } = useCallFeed(item.callFeed);
   const { rawDisplayName } = useRoomMemberName(member);
-  const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
+  const [tileRef, mediaRef] = useSpatialMediaStream(
+    stream,
+    audioOutputDevice,
+    audioContext,
+    isLocal
+  );
 
   // Firefox doesn't respect the disablePictureInPicture attribute
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@@ -57,6 +63,7 @@ export function VideoTileContainer({
       screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
       name={rawDisplayName}
       showName={showName}
+      ref={tileRef}
       mediaRef={mediaRef}
       avatar={getAvatar && getAvatar(member, width, height)}
       {...rest}

src/video-grid/useMediaStream.js

@@ -16,6 +16,8 @@ limitations under the License.
 
 import { useRef, useEffect } from "react";
 
+import { useSpatialAudio } from "../settings/useSetting";
+
 export function useMediaStream(stream, audioOutputDevice, mute = false) {
   const mediaRef = useRef();
 
@@ -74,3 +76,69 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
 
   return mediaRef;
 }
+
+export const useSpatialMediaStream = (
+  stream,
+  audioOutputDevice,
+  audioContext,
+  mute = false
+) => {
+  const tileRef = useRef();
+  const [spatialAudio] = useSpatialAudio();
+  // If spatial audio is enabled, we handle audio separately from the video element
+  const mediaRef = useMediaStream(
+    stream,
+    audioOutputDevice,
+    spatialAudio || mute
+  );
+
+  const pannerNodeRef = useRef();
+  if (!pannerNodeRef.current) {
+    pannerNodeRef.current = new PannerNode(audioContext, {
+      panningModel: "HRTF",
+      refDistance: 3,
+    });
+  }
+
+  const sourceRef = useRef();
+
+  useEffect(() => {
+    if (spatialAudio && tileRef.current && !mute) {
+      if (!sourceRef.current) {
+        sourceRef.current = audioContext.createMediaStreamSource(stream);
+      }
+
+      const tile = tileRef.current;
+      const source = sourceRef.current;
+      const pannerNode = pannerNodeRef.current;
+
+      const updatePosition = () => {
+        const bounds = tile.getBoundingClientRect();
+        const windowSize = Math.max(window.innerWidth, window.innerHeight);
+        // Position the source relative to its placement in the window
+        pannerNodeRef.current.positionX.value =
+          (bounds.x + bounds.width / 2) / windowSize - 0.5;
+        pannerNodeRef.current.positionY.value =
+          (bounds.y + bounds.height / 2) / windowSize - 0.5;
+        // Put the source in front of the listener
+        pannerNodeRef.current.positionZ.value = -2;
+      };
+
+      updatePosition();
+      source.connect(pannerNode);
+      pannerNode.connect(audioContext.destination);
+      // HACK: We abuse the CSS transitionrun event to detect when the tile
+      // moves, because useMeasure, IntersectionObserver, etc. all have no
+      // ability to track changes in the CSS transform property
+      tile.addEventListener("transitionrun", updatePosition);
+
+      return () => {
+        tile.removeEventListener("transitionrun", updatePosition);
+        source.disconnect();
+        pannerNode.disconnect();
+      };
+    }
+  }, [stream, spatialAudio, audioContext, mute]);
+
+  return [tileRef, mediaRef];
+};