xmj/element-call
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge pull request #736 from robintown/missing-audio-logs

Browse source 
Add logs to debug missing spatial audio
This commit is contained in:
Robin 2022-11-08 14:53:28 -05:00 • committed by GitHub
commit 57bf934bc2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 62 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/video-grid/VideoTileContainer.tsx
View file

@ -71,8 +71,8 @@ export function VideoTileContainer({
stream ?? null,
audioContext,
audioDestination,
isLocal || maximised,
localVolume
localVolume,
isLocal || maximised
);
const {
modalState: videoTileSettingsModalState,

109
src/video-grid/useMediaStream.ts
View file

@ -20,6 +20,7 @@ import {
acquireContext,
releaseContext,
} from "matrix-js-sdk/src/webrtc/audioContext";
import { logger } from "matrix-js-sdk/src/logger";
import { useSpatialAudio } from "../settings/useSetting";
import { useEventTarget } from "../useEvents";
@ -213,10 +214,10 @@ export const useSpatialMediaStream = (
stream: MediaStream | null,
audioContext: AudioContext,
audioDestination: AudioNode,
mute = false,
localVolume?: number
localVolume: number,
mute = false
): [RefObject<HTMLDivElement>, RefObject<MediaElement>] => {
const tileRef = useRef<HTMLDivElement>();
const tileRef = useRef<HTMLDivElement | null>(null);
const [spatialAudio] = useSpatialAudio();
// We always handle audio separately form the video element
const mediaRef = useMediaStream(stream, null, true);
@ -227,53 +228,63 @@ export const useSpatialMediaStream = (
const sourceRef = useRef<MediaStreamAudioSourceNode>();
useEffect(() => {
if (spatialAudio && tileRef.current && !mute && audioTrackCount > 0) {
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
refDistance: 3,
});
if (spatialAudio) {
if (tileRef.current && !mute && audioTrackCount > 0) {
logger.debug(`Rendering spatial audio for ${stream!.id}`);
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
refDistance: 3,
});
}
if (!gainNodeRef.current) {
gainNodeRef.current = new GainNode(audioContext, {
gain: localVolume,
});
}
if (!sourceRef.current || sourceRef.current.mediaStream !== stream!) {
sourceRef.current = audioContext.createMediaStreamSource(stream!);
}
const tile = tileRef.current;
const source = sourceRef.current;
const gainNode = gainNodeRef.current;
const pannerNode = pannerNodeRef.current;
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
// Position the source relative to its placement in the window
pannerNodeRef.current!.positionX.value =
(bounds.x + bounds.width / 2) / windowSize - 0.5;
pannerNodeRef.current!.positionY.value =
(bounds.y + bounds.height / 2) / windowSize - 0.5;
// Put the source in front of the listener
pannerNodeRef.current!.positionZ.value = -2;
};
gainNode.gain.value = localVolume;
updatePosition();
source.connect(gainNode).connect(pannerNode).connect(audioDestination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
// moves, because useMeasure, IntersectionObserver, etc. all have no
// ability to track changes in the CSS transform property
tile.addEventListener("transitionrun", updatePosition);
return () => {
tile.removeEventListener("transitionrun", updatePosition);
source.disconnect();
gainNode.disconnect();
pannerNode.disconnect();
};
} else if (stream) {
logger.debug(
`Not rendering spatial audio for ${stream.id} (tile ref ${Boolean(
tileRef.current
)}, mute ${mute}, track count ${audioTrackCount})`
);
}
if (!gainNodeRef.current) {
gainNodeRef.current = new GainNode(audioContext, {
gain: localVolume,
});
}
if (!sourceRef.current) {
sourceRef.current = audioContext.createMediaStreamSource(stream!);
}
const tile = tileRef.current;
const source = sourceRef.current;
const gainNode = gainNodeRef.current;
const pannerNode = pannerNodeRef.current;
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
// Position the source relative to its placement in the window
pannerNodeRef.current!.positionX.value =
(bounds.x + bounds.width / 2) / windowSize - 0.5;
pannerNodeRef.current!.positionY.value =
(bounds.y + bounds.height / 2) / windowSize - 0.5;
// Put the source in front of the listener
pannerNodeRef.current!.positionZ.value = -2;
};
gainNode.gain.value = localVolume;
updatePosition();
source.connect(gainNode).connect(pannerNode).connect(audioDestination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
// moves, because useMeasure, IntersectionObserver, etc. all have no
// ability to track changes in the CSS transform property
tile.addEventListener("transitionrun", updatePosition);
return () => {
tile.removeEventListener("transitionrun", updatePosition);
source.disconnect();
gainNode.disconnect();
pannerNode.disconnect();
};
}
}, [
stream,