xmj/element-call
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add logs to debug missing spatial audio

Browse source 
And potentially fix it, by recreating the source node when the stream changes.
This commit is contained in:
Robin Townsend 2022-11-07 11:41:49 -05:00
parent 85959046a5
commit 0a5701b9fa
2 changed files with 62 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/video-grid/VideoTileContainer.tsx
View file

@ -71,8 +71,8 @@ export function VideoTileContainer({
stream ?? null, stream ?? null,
audioContext, audioContext,
audioDestination, audioDestination,
isLocal, localVolume,
localVolume isLocal
); );
const { const {
modalState: videoTileSettingsModalState, modalState: videoTileSettingsModalState,

21
src/video-grid/useMediaStream.ts
View file

@ -20,6 +20,7 @@ import {
acquireContext, acquireContext,
releaseContext, releaseContext,
} from "matrix-js-sdk/src/webrtc/audioContext"; } from "matrix-js-sdk/src/webrtc/audioContext";
import { logger } from "matrix-js-sdk/src/logger";
import { useSpatialAudio } from "../settings/useSetting"; import { useSpatialAudio } from "../settings/useSetting";
import { useEventTarget } from "../useEvents"; import { useEventTarget } from "../useEvents";
@ -213,10 +214,10 @@ export const useSpatialMediaStream = (
stream: MediaStream | null, stream: MediaStream | null,
audioContext: AudioContext, audioContext: AudioContext,
audioDestination: AudioNode, audioDestination: AudioNode,
mute = false, localVolume: number,
localVolume?: number mute = false
): [RefObject<HTMLDivElement>, RefObject<MediaElement>] => { ): [RefObject<HTMLDivElement>, RefObject<MediaElement>] => {
const tileRef = useRef<HTMLDivElement>(); const tileRef = useRef<HTMLDivElement | null>(null);
const [spatialAudio] = useSpatialAudio(); const [spatialAudio] = useSpatialAudio();
// We always handle audio separately form the video element // We always handle audio separately form the video element
const mediaRef = useMediaStream(stream, null, true); const mediaRef = useMediaStream(stream, null, true);
@ -227,7 +228,10 @@ export const useSpatialMediaStream = (
const sourceRef = useRef<MediaStreamAudioSourceNode>(); const sourceRef = useRef<MediaStreamAudioSourceNode>();
useEffect(() => { useEffect(() => {
if (spatialAudio && tileRef.current && !mute && audioTrackCount > 0) { if (spatialAudio) {
if (tileRef.current && !mute && audioTrackCount > 0) {
logger.debug(`Rendering spatial audio for ${stream!.id}`);
if (!pannerNodeRef.current) { if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, { pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF", panningModel: "HRTF",
@ -239,7 +243,7 @@ export const useSpatialMediaStream = (
gain: localVolume, gain: localVolume,
}); });
} }
if (!sourceRef.current) { if (!sourceRef.current || sourceRef.current.mediaStream !== stream!) {
sourceRef.current = audioContext.createMediaStreamSource(stream!); sourceRef.current = audioContext.createMediaStreamSource(stream!);
} }
@ -274,6 +278,13 @@ export const useSpatialMediaStream = (
gainNode.disconnect(); gainNode.disconnect();
pannerNode.disconnect(); pannerNode.disconnect();
}; };
} else if (stream) {
logger.debug(
`Not rendering spatial audio for ${stream.id} (tile ref ${Boolean(
tileRef.current
)}, mute ${mute}, track count ${audioTrackCount})`
);
}
} }
}, [ }, [
stream, stream,