Fix full-screen audio

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>
This commit is contained in:
Šimon Brandner 2022-08-12 09:53:44 +02:00
parent 0e34f9a464
commit 8b97904144
No known key found for this signature in database
GPG key ID: D1D45825D60C24D2
4 changed files with 99 additions and 14 deletions

View file

@ -44,10 +44,11 @@ import { UserMenuContainer } from "../UserMenuContainer";
import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { useMediaHandler } from "../settings/useMediaHandler"; import { useMediaHandler } from "../settings/useMediaHandler";
import { useShowInspector } from "../settings/useSetting"; import { useShowInspector, useSpatialAudio } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal"; import { useModalTriggerState } from "../Modal";
import { useAudioContext } from "../video-grid/useMediaStream"; import { useAudioContext } from "../video-grid/useMediaStream";
import { useFullscreen } from "../video-grid/useFullscreen"; import { useFullscreen } from "../video-grid/useFullscreen";
import { AudioContainer } from "../video-grid/AudioContainer";
import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice"; import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@ -108,6 +109,8 @@ export function InCallView({
const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0); const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
const { toggleFullscreen, fullscreenParticipant } = useFullscreen(elementRef); const { toggleFullscreen, fullscreenParticipant } = useFullscreen(elementRef);
const [spatialAudio] = useSpatialAudio();
const [audioContext, audioDestination, audioRef] = useAudioContext(); const [audioContext, audioDestination, audioRef] = useAudioContext();
const { audioOutput } = useMediaHandler(); const { audioOutput } = useMediaHandler();
const [showInspector] = useShowInspector(); const [showInspector] = useShowInspector();
@ -183,7 +186,6 @@ export function InCallView({
key={fullscreenParticipant.id} key={fullscreenParticipant.id}
item={fullscreenParticipant} item={fullscreenParticipant}
getAvatar={renderAvatar} getAvatar={renderAvatar}
audioOutputDevice={audioOutput}
audioContext={audioContext} audioContext={audioContext}
audioDestination={audioDestination} audioDestination={audioDestination}
disableSpeakingIndicator={true} disableSpeakingIndicator={true}
@ -201,7 +203,6 @@ export function InCallView({
item={item} item={item}
getAvatar={renderAvatar} getAvatar={renderAvatar}
showName={items.length > 2 || item.focused} showName={items.length > 2 || item.focused}
audioOutputDevice={audioOutput}
audioContext={audioContext} audioContext={audioContext}
audioDestination={audioDestination} audioDestination={audioDestination}
disableSpeakingIndicator={items.length < 3} disableSpeakingIndicator={items.length < 3}
@ -217,7 +218,6 @@ export function InCallView({
items, items,
audioContext, audioContext,
audioDestination, audioDestination,
audioOutput,
layout, layout,
renderAvatar, renderAvatar,
toggleFullscreen, toggleFullscreen,
@ -235,6 +235,13 @@ export function InCallView({
return ( return (
<div className={styles.inRoom} ref={elementRef}> <div className={styles.inRoom} ref={elementRef}>
<audio ref={audioRef} /> <audio ref={audioRef} />
{(!spatialAudio || fullscreenParticipant) && (
<AudioContainer
items={items}
audioContext={audioContext}
audioDestination={audioDestination}
/>
)}
{!fullscreenParticipant && ( {!fullscreenParticipant && (
<Header> <Header>
<LeftNav> <LeftNav>

View file

@ -0,0 +1,85 @@
/*
Copyright 2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useEffect, useRef } from "react";
import { Participant } from "../room/InCallView";
import { useCallFeed } from "./useCallFeed";
// XXX: These in fact do not render anything but to my knowledge this is the
// only way to a hook on an array
interface AudioForParticipantProps {
item: Participant;
audioContext: AudioContext;
audioDestination: AudioNode;
}
export function AudioForParticipant({
item,
audioContext,
audioDestination,
}: AudioForParticipantProps): JSX.Element {
const { stream, localVolume } = useCallFeed(item.callFeed);
const gainNodeRef = useRef<GainNode>();
const sourceRef = useRef<MediaStreamAudioSourceNode>();
useEffect(() => {
if (!item.isLocal && stream.getAudioTracks().length > 0 && audioContext) {
if (!gainNodeRef.current) {
gainNodeRef.current = new GainNode(audioContext, {
gain: localVolume,
});
}
if (!sourceRef.current) {
sourceRef.current = audioContext.createMediaStreamSource(stream);
}
const source = sourceRef.current;
const gainNode = gainNodeRef.current;
gainNode.gain.value = localVolume;
source.connect(gainNode).connect(audioDestination);
return () => {
source.disconnect();
gainNode.disconnect();
};
}
}, [item, audioContext, audioDestination, stream, localVolume]);
return null;
}
interface AudioContainerProps {
items: Participant[];
audioContext: AudioContext;
audioDestination: AudioNode;
}
export function AudioContainer({
items,
...rest
}: AudioContainerProps): JSX.Element {
return (
<>
{items.map((item) => (
<AudioForParticipant key={item.id} item={item} {...rest} />
))}
</>
);
}

View file

@ -30,7 +30,6 @@ export function VideoTileContainer({
height, height,
getAvatar, getAvatar,
showName, showName,
audioOutputDevice,
audioContext, audioContext,
audioDestination, audioDestination,
disableSpeakingIndicator, disableSpeakingIndicator,
@ -52,7 +51,6 @@ export function VideoTileContainer({
const { rawDisplayName } = useRoomMemberName(member); const { rawDisplayName } = useRoomMemberName(member);
const [tileRef, mediaRef] = useSpatialMediaStream( const [tileRef, mediaRef] = useSpatialMediaStream(
stream, stream,
audioOutputDevice,
audioContext, audioContext,
audioDestination, audioDestination,
isLocal, isLocal,

View file

@ -177,7 +177,6 @@ export const useAudioContext = (): [
export const useSpatialMediaStream = ( export const useSpatialMediaStream = (
stream: MediaStream, stream: MediaStream,
audioOutputDevice: string,
audioContext: AudioContext, audioContext: AudioContext,
audioDestination: AudioNode, audioDestination: AudioNode,
mute = false, mute = false,
@ -185,13 +184,8 @@ export const useSpatialMediaStream = (
): [RefObject<Element>, RefObject<MediaElement>] => { ): [RefObject<Element>, RefObject<MediaElement>] => {
const tileRef = useRef<Element>(); const tileRef = useRef<Element>();
const [spatialAudio] = useSpatialAudio(); const [spatialAudio] = useSpatialAudio();
// If spatial audio is enabled, we handle audio separately from the video element // We always handle audio separately form the video element
const mediaRef = useMediaStream( const mediaRef = useMediaStream(stream, undefined, true, undefined);
stream,
audioOutputDevice,
spatialAudio || mute,
localVolume
);
const gainNodeRef = useRef<GainNode>(); const gainNodeRef = useRef<GainNode>();
const pannerNodeRef = useRef<PannerNode>(); const pannerNodeRef = useRef<PannerNode>();
@ -200,6 +194,7 @@ export const useSpatialMediaStream = (
useEffect(() => { useEffect(() => {
if ( if (
spatialAudio && spatialAudio &&
audioContext &&
tileRef.current && tileRef.current &&
!mute && !mute &&
stream.getAudioTracks().length > 0 stream.getAudioTracks().length > 0