Merge pull request #399 from robintown/chrome-spatial-aec

Make AEC work with spatial audio on Chrome

package.json

@@ -33,6 +33,7 @@
     "@sentry/react": "^6.13.3",
     "@sentry/tracing": "^6.13.3",
     "@types/grecaptcha": "^3.0.4",
+    "@types/sdp-transform": "^2.4.5",
     "@use-gesture/react": "^10.2.11",
     "classnames": "^2.3.1",
     "color-hash": "^2.0.1",
@@ -50,6 +51,7 @@
     "react-router-dom": "^5.2.0",
     "react-use-clipboard": "^1.0.7",
     "react-use-measure": "^2.1.1",
+    "sdp-transform": "^2.14.1",
     "unique-names-generator": "^4.6.0"
   },
   "devDependencies": {

src/@types/global.d.ts

@@ -21,4 +21,10 @@ declare global {
     // TODO: https://gitlab.matrix.org/matrix-org/olm/-/issues/10
     OLM_OPTIONS: Record<string, string>;
   }
+
+  // TypeScript doesn't know about the experimental setSinkId method, so we
+  // declare it ourselves
+  interface MediaElement extends HTMLMediaElement {
+    setSinkId: (id: string) => void;
+  }
 }

src/room/InCallView.jsx

@@ -42,6 +42,7 @@ import { usePreventScroll } from "@react-aria/overlays";
 import { useMediaHandler } from "../settings/useMediaHandler";
 import { useShowInspector } from "../settings/useSetting";
 import { useModalTriggerState } from "../Modal";
+import { useAudioContext } from "../video-grid/useMediaStream";
 
 const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -70,12 +71,10 @@ export function InCallView({
   usePreventScroll();
   const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
 
+  const [audioContext, audioDestination, audioRef] = useAudioContext();
   const { audioOutput } = useMediaHandler();
   const [showInspector] = useShowInspector();
 
-  const audioContext = useRef();
-  if (!audioContext.current) audioContext.current = new AudioContext();
-
   const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
     useModalTriggerState();
 
@@ -139,6 +138,7 @@ export function InCallView({
 
   return (
     <div className={styles.inRoom}>
+      <audio ref={audioRef} />
       <Header>
         <LeftNav>
           <RoomHeaderInfo roomName={roomName} avatarUrl={avatarUrl} />
@@ -165,7 +165,8 @@ export function InCallView({
               getAvatar={renderAvatar}
               showName={items.length > 2 || item.focused}
               audioOutputDevice={audioOutput}
-              audioContext={audioContext.current}
+              audioContext={audioContext}
+              audioDestination={audioDestination}
               disableSpeakingIndicator={items.length < 3}
               {...rest}
             />

src/video-grid/VideoTileContainer.jsx

@@ -29,6 +29,7 @@ export function VideoTileContainer({
   showName,
   audioOutputDevice,
   audioContext,
+  audioDestination,
   disableSpeakingIndicator,
   ...rest
 }) {
@@ -47,6 +48,7 @@ export function VideoTileContainer({
     stream,
     audioOutputDevice,
     audioContext,
+    audioDestination,
     isLocal
   );
 

src/video-grid/useMediaStream.ts

@@ -14,12 +14,24 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { useRef, useEffect } from "react";
+import { useRef, useEffect, RefObject } from "react";
+import { parse as parseSdp, write as writeSdp } from "sdp-transform";
 
 import { useSpatialAudio } from "../settings/useSetting";
 
-export function useMediaStream(stream, audioOutputDevice, mute = false) {
+declare global {
-  const mediaRef = useRef();
+  interface Window {
+    // For detecting whether this browser is Chrome or not
+    chrome?: unknown;
+  }
+}
+
+export const useMediaStream = (
+  stream: MediaStream,
+  audioOutputDevice: string,
+  mute = false
+): RefObject<MediaElement> => {
+  const mediaRef = useRef<MediaElement>();
 
   useEffect(() => {
     console.log(
@@ -75,15 +87,97 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
   }, []);
 
   return mediaRef;
+};
+
+// Loops the given audio stream back through a local peer connection, to make
+// AEC work with Web Audio streams on Chrome. The resulting stream should be
+// played through an audio element.
+// This hack can be removed once the following bug is resolved:
+// https://bugs.chromium.org/p/chromium/issues/detail?id=687574
+const createLoopback = async (stream: MediaStream): Promise<MediaStream> => {
+  // Prepare our local peer connections
+  const conn = new RTCPeerConnection();
+  const loopbackConn = new RTCPeerConnection();
+  const loopbackStream = new MediaStream();
+
+  conn.addEventListener("icecandidate", ({ candidate }) => {
+    if (candidate) loopbackConn.addIceCandidate(new RTCIceCandidate(candidate));
+  });
+  loopbackConn.addEventListener("icecandidate", ({ candidate }) => {
+    if (candidate) conn.addIceCandidate(new RTCIceCandidate(candidate));
+  });
+  loopbackConn.addEventListener("track", ({ track }) =>
+    loopbackStream.addTrack(track)
+  );
+
+  // Hook the connections together
+  stream.getTracks().forEach((track) => conn.addTrack(track));
+  const offer = await conn.createOffer({
+    offerToReceiveAudio: false,
+    offerToReceiveVideo: false,
+  });
+  await conn.setLocalDescription(offer);
+
+  await loopbackConn.setRemoteDescription(offer);
+  const answer = await loopbackConn.createAnswer();
+  // Rewrite SDP to be stereo and (variable) max bitrate
+  const parsedSdp = parseSdp(answer.sdp);
+  parsedSdp.media.forEach((m) =>
+    m.fmtp.forEach(
+      (f) => (f.config += `;stereo=1;cbr=0;maxaveragebitrate=510000;`)
+    )
+  );
+  answer.sdp = writeSdp(parsedSdp);
+
+  await loopbackConn.setLocalDescription(answer);
+  await conn.setRemoteDescription(answer);
+
+  return loopbackStream;
+};
+
+export const useAudioContext = (): [
+  AudioContext,
+  AudioNode,
+  RefObject<HTMLAudioElement>
+] => {
+  const context = useRef<AudioContext>();
+  const destination = useRef<AudioNode>();
+  const audioRef = useRef<HTMLAudioElement>();
+
+  useEffect(() => {
+    if (audioRef.current && !context.current) {
+      context.current = new AudioContext();
+
+      if (window.chrome) {
+        // We're in Chrome, which needs a loopback hack applied to enable AEC
+        const streamDest = context.current.createMediaStreamDestination();
+        destination.current = streamDest;
+
+        const audioEl = audioRef.current;
+        (async () => {
+          audioEl.srcObject = await createLoopback(streamDest.stream);
+          await audioEl.play();
+        })();
+        return () => {
+          audioEl.srcObject = null;
+        };
+      } else {
+        destination.current = context.current.destination;
       }
+    }
+  }, []);
+
+  return [context.current, destination.current, audioRef];
+};
 
 export const useSpatialMediaStream = (
-  stream,
+  stream: MediaStream,
-  audioOutputDevice,
+  audioOutputDevice: string,
-  audioContext,
+  audioContext: AudioContext,
+  audioDestination: AudioNode,
   mute = false
-) => {
+): [RefObject<Element>, RefObject<MediaElement>] => {
-  const tileRef = useRef();
+  const tileRef = useRef<Element>();
   const [spatialAudio] = useSpatialAudio();
   // If spatial audio is enabled, we handle audio separately from the video element
   const mediaRef = useMediaStream(
@@ -92,18 +186,17 @@ export const useSpatialMediaStream = (
     spatialAudio || mute
   );
 
-  const pannerNodeRef = useRef();
+  const pannerNodeRef = useRef<PannerNode>();
+  const sourceRef = useRef<MediaStreamAudioSourceNode>();
+
+  useEffect(() => {
+    if (spatialAudio && tileRef.current && !mute) {
       if (!pannerNodeRef.current) {
         pannerNodeRef.current = new PannerNode(audioContext, {
           panningModel: "HRTF",
           refDistance: 3,
         });
       }
-
-  const sourceRef = useRef();
-
-  useEffect(() => {
-    if (spatialAudio && tileRef.current && !mute) {
       if (!sourceRef.current) {
         sourceRef.current = audioContext.createMediaStreamSource(stream);
       }
@@ -125,8 +218,7 @@ export const useSpatialMediaStream = (
       };
 
       updatePosition();
-      source.connect(pannerNode);
+      source.connect(pannerNode).connect(audioDestination);
-      pannerNode.connect(audioContext.destination);
       // HACK: We abuse the CSS transitionrun event to detect when the tile
       // moves, because useMeasure, IntersectionObserver, etc. all have no
       // ability to track changes in the CSS transform property
@@ -138,7 +230,7 @@ export const useSpatialMediaStream = (
         pannerNode.disconnect();
       };
     }
-  }, [stream, spatialAudio, audioContext, mute]);
+  }, [stream, spatialAudio, audioContext, audioDestination, mute]);
 
   return [tileRef, mediaRef];
 };

yarn.lock

@@ -3024,6 +3024,11 @@
   resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39"
   integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==
 
+"@types/sdp-transform@^2.4.5":
+  version "2.4.5"
+  resolved "https://registry.yarnpkg.com/@types/sdp-transform/-/sdp-transform-2.4.5.tgz#3167961e0a1a5265545e278627aa37c606003f53"
+  integrity sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==
+
 "@types/source-list-map@*":
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9"
@@ -11046,6 +11051,11 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1:
     ajv "^6.12.5"
     ajv-keywords "^3.5.2"
 
+sdp-transform@^2.14.1:
+  version "2.14.1"
+  resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827"
+  integrity sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==
+
 "semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.6.0:
   version "5.7.1"
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"