Merge branch 'main' into vu-animation

src/room/GroupCallView.jsx

@@ -33,19 +33,6 @@ export function GroupCallView({
   roomId,
   groupCall,
 }) {
-  const [showInspector, setShowInspector] = useState(
-    () => !!localStorage.getItem("matrix-group-call-inspector")
-  );
-  const onChangeShowInspector = useCallback((show) => {
-    setShowInspector(show);
-
-    if (show) {
-      localStorage.setItem("matrix-group-call-inspector", "true");
-    } else {
-      localStorage.removeItem("matrix-group-call-inspector");
-    }
-  }, []);
-
   const {
     state,
     error,
@@ -104,8 +91,6 @@ export function GroupCallView({
           participants={participants}
           userMediaFeeds={userMediaFeeds}
           onLeave={onLeave}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
         />
       );
     } else {
@@ -126,8 +111,6 @@ export function GroupCallView({
           isScreensharing={isScreensharing}
           localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
-          setShowInspector={onChangeShowInspector}
-          showInspector={showInspector}
           roomId={roomId}
         />
       );
@@ -156,8 +139,6 @@ export function GroupCallView({
         localVideoMuted={localVideoMuted}
         toggleLocalVideoMuted={toggleLocalVideoMuted}
         toggleMicrophoneMuted={toggleMicrophoneMuted}
-        setShowInspector={onChangeShowInspector}
-        showInspector={showInspector}
         roomId={roomId}
       />
     );

src/room/InCallView.jsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useCallback, useMemo } from "react";
+import React, { useCallback, useMemo, useRef } from "react";
 import styles from "./InCallView.module.css";
 import {
   HangupButton,
@@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { usePreventScroll } from "@react-aria/overlays";
 import { useMediaHandler } from "../settings/useMediaHandler";
+import { useShowInspector } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
 
 const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@@ -57,14 +58,16 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   usePreventScroll();
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
 
   const { audioOutput } = useMediaHandler();
+  const [showInspector] = useShowInspector();
+
+  const audioContext = useRef();
+  if (!audioContext.current) audioContext.current = new AudioContext();
 
   const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
     useModalTriggerState();
@@ -151,6 +154,7 @@ export function InCallView({
               getAvatar={renderAvatar}
               showName={items.length > 2 || item.focused}
               audioOutputDevice={audioOutput}
+              audioContext={audioContext.current}
               disableSpeakingIndicator={items.length < 3}
               {...rest}
             />
@@ -169,8 +173,6 @@ export function InCallView({
         <OverflowMenu
           inCall
           roomId={roomId}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
           client={client}
           groupCall={groupCall}
           showInvite={true}

src/room/LobbyView.jsx

@@ -41,8 +41,6 @@ export function LobbyView({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   roomId,
 }) {
   const { stream } = useCallFeed(localCallFeed);
@@ -101,8 +99,6 @@ export function LobbyView({
               localVideoMuted={localVideoMuted}
               toggleLocalVideoMuted={toggleLocalVideoMuted}
               toggleMicrophoneMuted={toggleMicrophoneMuted}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               stream={stream}
               audioOutput={audioOutput}
             />

src/room/OverflowMenu.jsx

@@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
 
 export function OverflowMenu({
   roomId,
-  setShowInspector,
-  showInspector,
   inCall,
   groupCall,
   showInvite,
@@ -88,13 +86,7 @@ export function OverflowMenu({
           </Menu>
         )}
       </PopoverMenuTrigger>
-      {settingsModalState.isOpen && (
+      {settingsModalState.isOpen && <SettingsModal {...settingsModalProps} />}
-        <SettingsModal
-          {...settingsModalProps}
-          setShowInspector={setShowInspector}
-          showInspector={showInspector}
-        />
-      )}
       {inviteModalState.isOpen && (
         <InviteModal roomId={roomId} {...inviteModalProps} />
       )}

src/room/PTTButton.module.css

@@ -9,10 +9,12 @@
   background-color: #21262c;
   position: relative;
   padding: 0;
+  cursor: pointer;
 }
 
 .talking {
   background-color: #0dbd8b;
+  cursor: unset;
 }
 
 .error {

src/room/PTTCallView.tsx

@@ -44,8 +44,11 @@ function getPromptText(
   activeSpeakerIsLocalUser: boolean,
   talkOverEnabled: boolean,
   activeSpeakerUserId: string,
-  activeSpeakerDisplayName: string
+  activeSpeakerDisplayName: string,
+  connected: boolean
 ): string {
+  if (!connected) return "Connection Lost";
+
   const isTouchScreen = Boolean(window.ontouchstart !== undefined);
 
   if (showTalkOverError) {
@@ -84,8 +87,6 @@ interface Props {
   participants: RoomMember[];
   userMediaFeeds: CallFeed[];
   onLeave: () => void;
-  setShowInspector: (boolean) => void;
-  showInspector: boolean;
 }
 
 export const PTTCallView: React.FC<Props> = ({
@@ -97,8 +98,6 @@ export const PTTCallView: React.FC<Props> = ({
   participants,
   userMediaFeeds,
   onLeave,
-  setShowInspector,
-  showInspector,
 }) => {
   const { modalState: inviteModalState, modalProps: inviteModalProps } =
     useModalTriggerState();
@@ -128,6 +127,7 @@ export const PTTCallView: React.FC<Props> = ({
     startTalking,
     stopTalking,
     transmitBlocked,
+    connected,
   } = usePTT(
     client,
     groupCall,
@@ -190,8 +190,6 @@ export const PTTCallView: React.FC<Props> = ({
           <OverflowMenu
             inCall
             roomId={roomId}
-            setShowInspector={setShowInspector}
-            showInspector={showInspector}
             client={client}
             groupCall={groupCall}
             showInvite={false}
@@ -236,7 +234,8 @@ export const PTTCallView: React.FC<Props> = ({
               activeSpeakerIsLocalUser,
               talkOverEnabled,
               activeSpeakerUserId,
-              activeSpeakerDisplayName
+              activeSpeakerDisplayName,
+              connected
             )}
           </p>
           {userMediaFeeds.map((callFeed) => (

src/room/VideoPreview.jsx

@@ -35,8 +35,6 @@ export function VideoPreview({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
-  setShowInspector,
-  showInspector,
   audioOutput,
   stream,
 }) {
@@ -83,8 +81,6 @@ export function VideoPreview({
             />
             <OverflowMenu
               roomId={roomId}
-              setShowInspector={setShowInspector}
-              showInspector={showInspector}
               client={client}
               feedbackModalState={feedbackModalState}
               feedbackModalProps={feedbackModalProps}

src/room/useGroupCall.ts

@@ -18,10 +18,57 @@ import { useCallback, useEffect, useState } from "react";
 import {
   GroupCallEvent,
   GroupCallState,
+  GroupCall,
 } from "matrix-js-sdk/src/webrtc/groupCall";
+import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+import { RoomMember } from "matrix-js-sdk/src/models/room-member";
+
 import { usePageUnload } from "./usePageUnload";
 
-export function useGroupCall(groupCall) {
+export interface UseGroupCallType {
+  state: GroupCallState;
+  calls: MatrixCall[];
+  localCallFeed: CallFeed;
+  activeSpeaker: string;
+  userMediaFeeds: CallFeed[];
+  microphoneMuted: boolean;
+  localVideoMuted: boolean;
+  error: Error;
+  initLocalCallFeed: () => void;
+  enter: () => void;
+  leave: () => void;
+  toggleLocalVideoMuted: () => void;
+  toggleMicrophoneMuted: () => void;
+  toggleScreensharing: () => void;
+  requestingScreenshare: boolean;
+  isScreensharing: boolean;
+  screenshareFeeds: CallFeed[];
+  localScreenshareFeed: CallFeed;
+  localDesktopCapturerSourceId: string;
+  participants: RoomMember[];
+  hasLocalParticipant: boolean;
+}
+
+interface State {
+  state: GroupCallState;
+  calls: MatrixCall[];
+  localCallFeed: CallFeed;
+  activeSpeaker: string;
+  userMediaFeeds: CallFeed[];
+  error: Error;
+  microphoneMuted: boolean;
+  localVideoMuted: boolean;
+  screenshareFeeds: CallFeed[];
+  localScreenshareFeed: CallFeed;
+  localDesktopCapturerSourceId: string;
+  isScreensharing: boolean;
+  requestingScreenshare: boolean;
+  participants: RoomMember[];
+  hasLocalParticipant: boolean;
+}
+
+export function useGroupCall(groupCall: GroupCall): UseGroupCallType {
   const [
     {
       state,
@@ -41,20 +88,25 @@ export function useGroupCall(groupCall) {
       requestingScreenshare,
     },
     setState,
-  ] = useState({
+  ] = useState<State>({
     state: GroupCallState.LocalCallFeedUninitialized,
     calls: [],
+    localCallFeed: null,
+    activeSpeaker: null,
     userMediaFeeds: [],
+    error: null,
     microphoneMuted: false,
     localVideoMuted: false,
-    screenshareFeeds: [],
     isScreensharing: false,
+    screenshareFeeds: [],
+    localScreenshareFeed: null,
+    localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
     hasLocalParticipant: false,
   });
 
-  const updateState = (state) =>
+  const updateState = (state: Partial<State>) =>
     setState((prevState) => ({ ...prevState, ...state }));
 
   useEffect(() => {
@@ -75,25 +127,28 @@ export function useGroupCall(groupCall) {
       });
     }
 
-    function onUserMediaFeedsChanged(userMediaFeeds) {
+    function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void {
       updateState({
         userMediaFeeds: [...userMediaFeeds],
       });
     }
 
-    function onScreenshareFeedsChanged(screenshareFeeds) {
+    function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void {
       updateState({
         screenshareFeeds: [...screenshareFeeds],
       });
     }
 
-    function onActiveSpeakerChanged(activeSpeaker) {
+    function onActiveSpeakerChanged(activeSpeaker: string): void {
       updateState({
         activeSpeaker: activeSpeaker,
       });
     }
 
-    function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) {
+    function onLocalMuteStateChanged(
+      microphoneMuted: boolean,
+      localVideoMuted: boolean
+    ): void {
       updateState({
         microphoneMuted,
         localVideoMuted,
@@ -101,10 +156,10 @@ export function useGroupCall(groupCall) {
     }
 
     function onLocalScreenshareStateChanged(
-      isScreensharing,
+      isScreensharing: boolean,
-      localScreenshareFeed,
+      localScreenshareFeed: CallFeed,
-      localDesktopCapturerSourceId
+      localDesktopCapturerSourceId: string
-    ) {
+    ): void {
       updateState({
         isScreensharing,
         localScreenshareFeed,
@@ -112,13 +167,13 @@ export function useGroupCall(groupCall) {
       });
     }
 
-    function onCallsChanged(calls) {
+    function onCallsChanged(calls: MatrixCall[]): void {
       updateState({
         calls: [...calls],
       });
     }
 
-    function onParticipantsChanged(participants) {
+    function onParticipantsChanged(participants: RoomMember[]): void {
       updateState({
         participants: [...participants],
         hasLocalParticipant: groupCall.hasLocalParticipant(),

src/room/usePTT.ts

@@ -15,10 +15,11 @@ limitations under the License.
 */
 
 import { useCallback, useEffect, useState } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
 import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
 import { logger } from "matrix-js-sdk/src/logger";
+import { SyncState } from "matrix-js-sdk/src/sync";
 
 import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
 
@@ -30,6 +31,21 @@ function getActiveSpeakerFeed(
 ): CallFeed | null {
   const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
 
+  // make sure the feeds are in a deterministic order so every client picks
+  // the same one as the active speaker. The custom sort function sorts
+  // by user ID, so needs a collator of some kind to compare. We make a
+  // specific one to help ensure every client sorts the same way
+  // although of course user IDs shouldn't contain accented characters etc.
+  // anyway).
+  const collator = new Intl.Collator("en", {
+    sensitivity: "variant",
+    usage: "sort",
+    ignorePunctuation: false,
+  });
+  activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number =>
+    collator.compare(a.userId, b.userId)
+  );
+
   let activeSpeakerFeed = null;
   let highestPowerLevel = null;
   for (const feed of activeSpeakerFeeds) {
@@ -53,6 +69,11 @@ export interface PTTState {
   startTalking: () => void;
   stopTalking: () => void;
   transmitBlocked: boolean;
+  // connected is actually an indication of whether we're connected to the HS
+  // (ie. the client's syncing state) rather than media connection, since
+  // it's peer to peer so we can't really say which peer is 'disconnected' if
+  // there's only one other person in the call and they've lost Internet.
+  connected: boolean;
 }
 
 export const usePTT = (
@@ -226,6 +247,17 @@ export const usePTT = (
     setMicMuteWrapper(true);
   }, [setMicMuteWrapper]);
 
+  // separate state for connected: we set it separately from other things
+  // in the client sync callback
+  const [connected, setConnected] = useState(true);
+
+  const onClientSync = useCallback(
+    (syncState: SyncState) => {
+      setConnected(syncState !== SyncState.Error);
+    },
+    [setConnected]
+  );
+
   useEffect(() => {
     function onKeyDown(event: KeyboardEvent): void {
       if (event.code === "Space") {
@@ -275,8 +307,18 @@ export const usePTT = (
     pttButtonHeld,
     enablePTTButton,
     setMicMuteWrapper,
+    client,
+    onClientSync,
   ]);
 
+  useEffect(() => {
+    client.on(ClientEvent.Sync, onClientSync);
+
+    return () => {
+      client.removeListener(ClientEvent.Sync, onClientSync);
+    };
+  }, [client, onClientSync]);
+
   const setTalkOverEnabled = useCallback((talkOverEnabled) => {
     setState((prevState) => ({
       ...prevState,
@@ -294,5 +336,6 @@ export const usePTT = (
     startTalking,
     stopTalking,
     transmitBlocked,
+    connected,
   };
 };

src/settings/SettingsModal.jsx

@@ -24,12 +24,13 @@ import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
 import { SelectInput } from "../input/SelectInput";
 import { Item } from "@react-stately/collections";
 import { useMediaHandler } from "./useMediaHandler";
+import { useSpatialAudio, useShowInspector } from "./useSetting";
 import { FieldRow, InputField } from "../input/Input";
 import { Button } from "../button";
 import { useDownloadDebugLog } from "./submit-rageshake";
 import { Body } from "../typography/Typography";
 
-export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
+export const SettingsModal = (props) => {
   const {
     audioInput,
     audioInputs,
@@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
     audioOutputs,
     setAudioOutput,
   } = useMediaHandler();
+  const [spatialAudio, setSpatialAudio] = useSpatialAudio();
+  const [showInspector, setShowInspector] = useShowInspector();
 
   const downloadDebugLog = useDownloadDebugLog();
 
@@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       isDismissable
       mobileFullScreen
       className={styles.settingsModal}
-      {...rest}
+      {...props}
     >
       <TabContainer className={styles.tabContainer}>
         <TabItem
@@ -81,6 +84,15 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
               ))}
             </SelectInput>
           )}
+          <FieldRow>
+            <InputField
+              id="spatialAudio"
+              label="Spatial audio (experimental)"
+              type="checkbox"
+              checked={spatialAudio}
+              onChange={(e) => setSpatialAudio(e.target.checked)}
+            />
+          </FieldRow>
         </TabItem>
         <TabItem
           title={
@@ -130,4 +142,4 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
       </TabContainer>
     </Modal>
   );
-}
+};

src/settings/useSetting.ts

@@ -0,0 +1,56 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { useMemo, useState, useEffect, useCallback } from "react";
+
+// Bus to notify other useSetting consumers when a setting is changed
+const settingsBus = new EventEmitter();
+
+// Like useState, but reads from and persists the value to localStorage
+const useSetting = <T>(
+  name: string,
+  defaultValue: T
+): [T, (value: T) => void] => {
+  const key = useMemo(() => `matrix-setting-${name}`, [name]);
+
+  const [value, setValue] = useState<T>(() => {
+    const item = localStorage.getItem(key);
+    return item == null ? defaultValue : JSON.parse(item);
+  });
+
+  useEffect(() => {
+    settingsBus.on(name, setValue);
+    return () => {
+      settingsBus.off(name, setValue);
+    };
+  }, [name, setValue]);
+
+  return [
+    value,
+    useCallback(
+      (newValue: T) => {
+        setValue(newValue);
+        localStorage.setItem(key, JSON.stringify(newValue));
+        settingsBus.emit(name, newValue);
+      },
+      [name, key, setValue]
+    ),
+  ];
+};
+
+export const useSpatialAudio = () => useSetting("spatial-audio", false);
+export const useShowInspector = () => useSetting("show-inspector", false);

src/video-grid/VideoTile.jsx

@@ -14,14 +14,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, { forwardRef } from "react";
 import { animated } from "@react-spring/web";
 import classNames from "classnames";
 import styles from "./VideoTile.module.css";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
 
-export function VideoTile({
+export const VideoTile = forwardRef(
+  (
+    {
       className,
       isLocal,
       speaking,
@@ -34,7 +36,9 @@ export function VideoTile({
       showName,
       mediaRef,
       ...rest
-}) {
+    },
+    ref
+  ) => {
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -43,6 +47,7 @@ export function VideoTile({
           [styles.muted]: audioMuted,
           [styles.screenshare]: screenshare,
         })}
+        ref={ref}
         {...rest}
       >
         {(videoMuted || noVideo) && (
@@ -68,3 +73,4 @@ export function VideoTile({
       </animated.div>
     );
   }
+);

src/video-grid/VideoTile.module.css

@@ -5,6 +5,10 @@
   overflow: hidden;
   cursor: pointer;
   touch-action: none;
+
+  /* HACK: This has no visual effect due to the short duration, but allows the
+  JS to detect movement via the transform property for audio spatialization */
+  transition: transform 0.000000001s;
 }
 
 .videoTile * {

src/video-grid/VideoTileContainer.jsx

@@ -17,7 +17,7 @@ limitations under the License.
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 import React from "react";
 import { useCallFeed } from "./useCallFeed";
-import { useMediaStream } from "./useMediaStream";
+import { useSpatialMediaStream } from "./useMediaStream";
 import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 
@@ -28,6 +28,7 @@ export function VideoTileContainer({
   getAvatar,
   showName,
   audioOutputDevice,
+  audioContext,
   disableSpeakingIndicator,
   ...rest
 }) {
@@ -42,7 +43,12 @@ export function VideoTileContainer({
     member,
   } = useCallFeed(item.callFeed);
   const { rawDisplayName } = useRoomMemberName(member);
-  const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
+  const [tileRef, mediaRef] = useSpatialMediaStream(
+    stream,
+    audioOutputDevice,
+    audioContext,
+    isLocal
+  );
 
   // Firefox doesn't respect the disablePictureInPicture attribute
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@@ -57,6 +63,7 @@ export function VideoTileContainer({
       screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
       name={rawDisplayName}
       showName={showName}
+      ref={tileRef}
       mediaRef={mediaRef}
       avatar={getAvatar && getAvatar(member, width, height)}
       {...rest}

src/video-grid/useMediaStream.js

@@ -16,6 +16,8 @@ limitations under the License.
 
 import { useRef, useEffect } from "react";
 
+import { useSpatialAudio } from "../settings/useSetting";
+
 export function useMediaStream(stream, audioOutputDevice, mute = false) {
   const mediaRef = useRef();
 
@@ -55,7 +57,8 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
       mediaRef.current !== undefined
     ) {
       console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
-      mediaRef.current.setSinkId(audioOutputDevice);
+      // Chrome for Android doesn't support this
+      mediaRef.current.setSinkId?.(audioOutputDevice);
     }
   }, [audioOutputDevice]);
 
@@ -73,3 +76,69 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
 
   return mediaRef;
 }
+
+export const useSpatialMediaStream = (
+  stream,
+  audioOutputDevice,
+  audioContext,
+  mute = false
+) => {
+  const tileRef = useRef();
+  const [spatialAudio] = useSpatialAudio();
+  // If spatial audio is enabled, we handle audio separately from the video element
+  const mediaRef = useMediaStream(
+    stream,
+    audioOutputDevice,
+    spatialAudio || mute
+  );
+
+  const pannerNodeRef = useRef();
+  if (!pannerNodeRef.current) {
+    pannerNodeRef.current = new PannerNode(audioContext, {
+      panningModel: "HRTF",
+      refDistance: 3,
+    });
+  }
+
+  const sourceRef = useRef();
+
+  useEffect(() => {
+    if (spatialAudio && tileRef.current && !mute) {
+      if (!sourceRef.current) {
+        sourceRef.current = audioContext.createMediaStreamSource(stream);
+      }
+
+      const tile = tileRef.current;
+      const source = sourceRef.current;
+      const pannerNode = pannerNodeRef.current;
+
+      const updatePosition = () => {
+        const bounds = tile.getBoundingClientRect();
+        const windowSize = Math.max(window.innerWidth, window.innerHeight);
+        // Position the source relative to its placement in the window
+        pannerNodeRef.current.positionX.value =
+          (bounds.x + bounds.width / 2) / windowSize - 0.5;
+        pannerNodeRef.current.positionY.value =
+          (bounds.y + bounds.height / 2) / windowSize - 0.5;
+        // Put the source in front of the listener
+        pannerNodeRef.current.positionZ.value = -2;
+      };
+
+      updatePosition();
+      source.connect(pannerNode);
+      pannerNode.connect(audioContext.destination);
+      // HACK: We abuse the CSS transitionrun event to detect when the tile
+      // moves, because useMeasure, IntersectionObserver, etc. all have no
+      // ability to track changes in the CSS transform property
+      tile.addEventListener("transitionrun", updatePosition);
+
+      return () => {
+        tile.removeEventListener("transitionrun", updatePosition);
+        source.disconnect();
+        pannerNode.disconnect();
+      };
+    }
+  }, [stream, spatialAudio, audioContext, mute]);
+
+  return [tileRef, mediaRef];
+};