From ee43fcc91fffeecafb618a6b73082d3a8a01b937 Mon Sep 17 00:00:00 2001
From: Robin Townsend <robin@robin.town>
Date: Mon, 13 Jun 2022 13:31:44 -0400
Subject: [PATCH] Make AEC work with spatial audio on Chrome

---
 package.json                          |  1 +
 src/room/InCallView.jsx               |  9 +--
 src/video-grid/VideoTileContainer.jsx |  2 +
 src/video-grid/useMediaStream.js      | 93 ++++++++++++++++++++++++---
 yarn.lock                             |  5 ++
 5 files changed, 97 insertions(+), 13 deletions(-)

diff --git a/package.json b/package.json
index 4f0309e..9c4b8fc 100644
--- a/package.json
+++ b/package.json
@@ -50,6 +50,7 @@
     "react-router-dom": "^5.2.0",
     "react-use-clipboard": "^1.0.7",
     "react-use-measure": "^2.1.1",
+    "sdp-transform": "^2.14.1",
     "unique-names-generator": "^4.6.0"
   },
   "devDependencies": {
diff --git a/src/room/InCallView.jsx b/src/room/InCallView.jsx
index de9f5bc..4a4c1cb 100644
--- a/src/room/InCallView.jsx
+++ b/src/room/InCallView.jsx
@@ -42,6 +42,7 @@ import { usePreventScroll } from "@react-aria/overlays";
 import { useMediaHandler } from "../settings/useMediaHandler";
 import { useShowInspector } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
+import { useAudioContext } from "../video-grid/useMediaStream";
 
 const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -70,12 +71,10 @@ export function InCallView({
   usePreventScroll();
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
 
+  const [audioContext, audioDestination, audioRef] = useAudioContext();
   const { audioOutput } = useMediaHandler();
   const [showInspector] = useShowInspector();
 
-  const audioContext = useRef();
-  if (!audioContext.current) audioContext.current = new AudioContext();
-
   const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
     useModalTriggerState();
 
@@ -139,6 +138,7 @@ export function InCallView({
 
   return (
     <div className={styles.inRoom}>
+      <audio ref={audioRef} />
       <Header>
         <LeftNav>
           <RoomHeaderInfo roomName={roomName} avatarUrl={avatarUrl} />
@@ -165,7 +165,8 @@ export function InCallView({
               getAvatar={renderAvatar}
               showName={items.length > 2 || item.focused}
               audioOutputDevice={audioOutput}
-              audioContext={audioContext.current}
+              audioContext={audioContext}
+              audioDestination={audioDestination}
               disableSpeakingIndicator={items.length < 3}
               {...rest}
             />
diff --git a/src/video-grid/VideoTileContainer.jsx b/src/video-grid/VideoTileContainer.jsx
index a4cf400..8fb4f80 100644
--- a/src/video-grid/VideoTileContainer.jsx
+++ b/src/video-grid/VideoTileContainer.jsx
@@ -29,6 +29,7 @@ export function VideoTileContainer({
   showName,
   audioOutputDevice,
   audioContext,
+  audioDestination,
   disableSpeakingIndicator,
   ...rest
 }) {
@@ -47,6 +48,7 @@ export function VideoTileContainer({
     stream,
     audioOutputDevice,
     audioContext,
+    audioDestination,
     isLocal
   );
 
diff --git a/src/video-grid/useMediaStream.js b/src/video-grid/useMediaStream.js
index 774c6dd..5622441 100644
--- a/src/video-grid/useMediaStream.js
+++ b/src/video-grid/useMediaStream.js
@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import { useRef, useEffect } from "react";
+import { parse as parseSdp, write as writeSdp } from "sdp-transform";
 
 import { useSpatialAudio } from "../settings/useSetting";
 
@@ -77,10 +78,85 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
   return mediaRef;
 }
 
+// Loops the given audio stream back through a local peer connection, to make
+// AEC work with Web Audio streams on Chrome. The resulting stream should be
+// played through an audio element.
+// This hack can be removed once the following bug is resolved:
+// https://bugs.chromium.org/p/chromium/issues/detail?id=687574
+const createLoopback = async (stream) => {
+  // Prepare our local peer connections
+  const conn = new RTCPeerConnection();
+  const loopbackConn = new RTCPeerConnection();
+  const loopbackStream = new MediaStream();
+
+  conn.addEventListener("icecandidate", ({ candidate }) => {
+    if (candidate) loopbackConn.addIceCandidate(new RTCIceCandidate(candidate));
+  });
+  loopbackConn.addEventListener("icecandidate", ({ candidate }) => {
+    if (candidate) conn.addIceCandidate(new RTCIceCandidate(candidate));
+  });
+  loopbackConn.addEventListener("track", ({ track }) =>
+    loopbackStream.addTrack(track),
+  );
+
+  // Hook the connections together
+  stream.getTracks().forEach((track) => conn.addTrack(track));
+  const offer = await conn.createOffer({
+    offerVideo: false,
+    offerAudio: true,
+    offerToReceiveAudio: false,
+    offerToReceiveVideo: false,
+  });
+  await conn.setLocalDescription(offer);
+
+  await loopbackConn.setRemoteDescription(offer);
+  const answer = await loopbackConn.createAnswer();
+  // Rewrite SDP to be stereo and (variable) max bitrate
+  const parsedSdp = parseSdp(answer.sdp);
+  parsedSdp.media.forEach((m) => m.fmtp.forEach((f) =>
+    f.config += `;stereo=1;cbr=0;maxaveragebitrate=510000;`),
+  );
+  answer.sdp = writeSdp(parsedSdp);
+
+  await loopbackConn.setLocalDescription(answer);
+  await conn.setRemoteDescription(answer);
+
+  return loopbackStream;
+};
+
+export const useAudioContext = () => {
+  const context = useRef();
+  const destination = useRef();
+  const audioRef = useRef();
+
+  useEffect(() => {
+    if (audioRef.current && !context.current) {
+      context.current = new AudioContext();
+
+      if (window.chrome) {
+        // We're in Chrome, which needs a loopback hack applied to enable AEC
+        destination.current = context.current.createMediaStreamDestination();
+
+        const audioEl = audioRef.current;
+        (async () => {
+          audioEl.srcObject = await createLoopback(destination.current.stream);
+          await audioEl.play();
+        })();
+        return () => { audioEl.srcObject = null; };
+      } else {
+        destination.current = context.current.destination;
+      }
+    }
+  }, []);
+
+  return [context.current, destination.current, audioRef];
+};
+
 export const useSpatialMediaStream = (
   stream,
   audioOutputDevice,
   audioContext,
+  audioDestination,
   mute = false
 ) => {
   const tileRef = useRef();
@@ -93,17 +169,16 @@ export const useSpatialMediaStream = (
   );
 
   const pannerNodeRef = useRef();
-  if (!pannerNodeRef.current) {
-    pannerNodeRef.current = new PannerNode(audioContext, {
-      panningModel: "HRTF",
-      refDistance: 3,
-    });
-  }
-
   const sourceRef = useRef();
 
   useEffect(() => {
     if (spatialAudio && tileRef.current && !mute) {
+      if (!pannerNodeRef.current) {
+        pannerNodeRef.current = new PannerNode(audioContext, {
+          panningModel: "HRTF",
+          refDistance: 3,
+        });
+      }
       if (!sourceRef.current) {
         sourceRef.current = audioContext.createMediaStreamSource(stream);
       }
@@ -126,7 +201,7 @@ export const useSpatialMediaStream = (
 
       updatePosition();
       source.connect(pannerNode);
-      pannerNode.connect(audioContext.destination);
+      pannerNode.connect(audioDestination);
       // HACK: We abuse the CSS transitionrun event to detect when the tile
       // moves, because useMeasure, IntersectionObserver, etc. all have no
       // ability to track changes in the CSS transform property
@@ -138,7 +213,7 @@ export const useSpatialMediaStream = (
         pannerNode.disconnect();
       };
     }
-  }, [stream, spatialAudio, audioContext, mute]);
+  }, [stream, spatialAudio, audioContext, audioDestination, mute]);
 
   return [tileRef, mediaRef];
 };
diff --git a/yarn.lock b/yarn.lock
index 5d3ab8d..fa83a1f 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -11046,6 +11046,11 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
+sdp-transform@^2.14.1:
+  version "2.14.1"
+  resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827"
+  integrity sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==
+
 "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"