diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx
index 0e9cba1..7037765 100644
--- a/src/room/InCallView.tsx
+++ b/src/room/InCallView.tsx
@@ -44,10 +44,11 @@ import { UserMenuContainer } from "../UserMenuContainer";
 import { useRageshakeRequestModal } from "../settings/submit-rageshake";
 import { RageshakeRequestModal } from "./RageshakeRequestModal";
 import { useMediaHandler } from "../settings/useMediaHandler";
-import { useShowInspector } from "../settings/useSetting";
+import { useShowInspector, useSpatialAudio } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
 import { useAudioContext } from "../video-grid/useMediaStream";
 import { useFullscreen } from "../video-grid/useFullscreen";
+import { AudioContainer } from "../video-grid/AudioContainer";
 import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -108,6 +109,8 @@ export function InCallView({
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
   const { toggleFullscreen, fullscreenParticipant } = useFullscreen(elementRef);
 
+  const [spatialAudio] = useSpatialAudio();
+
   const [audioContext, audioDestination, audioRef] = useAudioContext();
   const { audioOutput } = useMediaHandler();
   const [showInspector] = useShowInspector();
@@ -183,7 +186,6 @@ export function InCallView({
           key={fullscreenParticipant.id}
           item={fullscreenParticipant}
           getAvatar={renderAvatar}
-          audioOutputDevice={audioOutput}
           audioContext={audioContext}
           audioDestination={audioDestination}
           disableSpeakingIndicator={true}
@@ -201,7 +203,6 @@ export function InCallView({
             item={item}
             getAvatar={renderAvatar}
             showName={items.length > 2 || item.focused}
-            audioOutputDevice={audioOutput}
             audioContext={audioContext}
             audioDestination={audioDestination}
             disableSpeakingIndicator={items.length < 3}
@@ -217,7 +218,6 @@ export function InCallView({
     items,
     audioContext,
     audioDestination,
-    audioOutput,
     layout,
     renderAvatar,
     toggleFullscreen,
@@ -235,6 +235,13 @@ export function InCallView({
   return (
     <div className={styles.inRoom} ref={elementRef}>
       <audio ref={audioRef} />
+      {(!spatialAudio || fullscreenParticipant) && (
+        <AudioContainer
+          items={items}
+          audioContext={audioContext}
+          audioDestination={audioDestination}
+        />
+      )}
       {!fullscreenParticipant && (
         <Header>
           <LeftNav>
diff --git a/src/video-grid/AudioContainer.tsx b/src/video-grid/AudioContainer.tsx
new file mode 100644
index 0000000..d756d00
--- /dev/null
+++ b/src/video-grid/AudioContainer.tsx
@@ -0,0 +1,85 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, { useEffect, useRef } from "react";
+
+import { Participant } from "../room/InCallView";
+import { useCallFeed } from "./useCallFeed";
+
+// XXX: These in fact do not render anything but to my knowledge this is the
+// only way to a hook on an array
+
+interface AudioForParticipantProps {
+  item: Participant;
+  audioContext: AudioContext;
+  audioDestination: AudioNode;
+}
+
+export function AudioForParticipant({
+  item,
+  audioContext,
+  audioDestination,
+}: AudioForParticipantProps): JSX.Element {
+  const { stream, localVolume } = useCallFeed(item.callFeed);
+
+  const gainNodeRef = useRef<GainNode>();
+  const sourceRef = useRef<MediaStreamAudioSourceNode>();
+
+  useEffect(() => {
+    if (!item.isLocal && stream.getAudioTracks().length > 0 && audioContext) {
+      if (!gainNodeRef.current) {
+        gainNodeRef.current = new GainNode(audioContext, {
+          gain: localVolume,
+        });
+      }
+      if (!sourceRef.current) {
+        sourceRef.current = audioContext.createMediaStreamSource(stream);
+      }
+
+      const source = sourceRef.current;
+      const gainNode = gainNodeRef.current;
+
+      gainNode.gain.value = localVolume;
+      source.connect(gainNode).connect(audioDestination);
+
+      return () => {
+        source.disconnect();
+        gainNode.disconnect();
+      };
+    }
+  }, [item, audioContext, audioDestination, stream, localVolume]);
+
+  return null;
+}
+
+interface AudioContainerProps {
+  items: Participant[];
+  audioContext: AudioContext;
+  audioDestination: AudioNode;
+}
+
+export function AudioContainer({
+  items,
+  ...rest
+}: AudioContainerProps): JSX.Element {
+  return (
+    <>
+      {items.map((item) => (
+        <AudioForParticipant key={item.id} item={item} {...rest} />
+      ))}
+    </>
+  );
+}
diff --git a/src/video-grid/VideoTileContainer.jsx b/src/video-grid/VideoTileContainer.jsx
index 55de199..719d46e 100644
--- a/src/video-grid/VideoTileContainer.jsx
+++ b/src/video-grid/VideoTileContainer.jsx
@@ -30,7 +30,6 @@ export function VideoTileContainer({
   height,
   getAvatar,
   showName,
-  audioOutputDevice,
   audioContext,
   audioDestination,
   disableSpeakingIndicator,
@@ -52,7 +51,6 @@ export function VideoTileContainer({
   const { rawDisplayName } = useRoomMemberName(member);
   const [tileRef, mediaRef] = useSpatialMediaStream(
     stream,
-    audioOutputDevice,
     audioContext,
     audioDestination,
     isLocal,
diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts
index dc915f1..4646a2c 100644
--- a/src/video-grid/useMediaStream.ts
+++ b/src/video-grid/useMediaStream.ts
@@ -177,7 +177,6 @@ export const useAudioContext = (): [
 
 export const useSpatialMediaStream = (
   stream: MediaStream,
-  audioOutputDevice: string,
   audioContext: AudioContext,
   audioDestination: AudioNode,
   mute = false,
@@ -185,13 +184,8 @@ export const useSpatialMediaStream = (
 ): [RefObject<Element>, RefObject<MediaElement>] => {
   const tileRef = useRef<Element>();
   const [spatialAudio] = useSpatialAudio();
-  // If spatial audio is enabled, we handle audio separately from the video element
-  const mediaRef = useMediaStream(
-    stream,
-    audioOutputDevice,
-    spatialAudio || mute,
-    localVolume
-  );
+  // We always handle audio separately form the video element
+  const mediaRef = useMediaStream(stream, undefined, true, undefined);
 
   const gainNodeRef = useRef<GainNode>();
   const pannerNodeRef = useRef<PannerNode>();
@@ -200,6 +194,7 @@ export const useSpatialMediaStream = (
   useEffect(() => {
     if (
       spatialAudio &&
+      audioContext &&
       tileRef.current &&
       !mute &&
       stream.getAudioTracks().length > 0