xmj/element-call
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge remote-tracking branch 'origin/main' into dbkr/avoid-browser-index-import

Browse source
This commit is contained in:
David Baker 2022-06-01 16:03:18 +01:00
commit 5282ab5f12
17 changed files with 396 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
src/index.css
View file

@ -26,6 +26,8 @@ limitations under the License.
--inter-unicode-range: U+0000-20e2, U+20e4-23ce, U+23d0-24c1, U+24c3-259f,
U+25c2-2664, U+2666-2763, U+2765-2b05, U+2b07-2b1b, U+2b1d-10FFFF;
--primaryColor: #0dbd8b;
--primaryColor-20: #0dbd8b33;
--alert-20: #ff5b5533;
--bgColor1: #15191e;
--bgColor2: #21262c;
--bgColor3: #444;

19
src/room/GroupCallView.jsx
View file

@ -33,19 +33,6 @@ export function GroupCallView({
roomId,
groupCall,
}) {
const [showInspector, setShowInspector] = useState(
() => !!localStorage.getItem("matrix-group-call-inspector")
);
const onChangeShowInspector = useCallback((show) => {
setShowInspector(show);
if (show) {
localStorage.setItem("matrix-group-call-inspector", "true");
} else {
localStorage.removeItem("matrix-group-call-inspector");
}
}, []);
const {
state,
error,
@ -104,8 +91,6 @@ export function GroupCallView({
participants={participants}
userMediaFeeds={userMediaFeeds}
onLeave={onLeave}
setShowInspector={onChangeShowInspector}
showInspector={showInspector}
/>
);
} else {
@ -126,8 +111,6 @@ export function GroupCallView({
isScreensharing={isScreensharing}
localScreenshareFeed={localScreenshareFeed}
screenshareFeeds={screenshareFeeds}
setShowInspector={onChangeShowInspector}
showInspector={showInspector}
roomId={roomId}
/>
);
@ -156,8 +139,6 @@ export function GroupCallView({
localVideoMuted={localVideoMuted}
toggleLocalVideoMuted={toggleLocalVideoMuted}
toggleMicrophoneMuted={toggleMicrophoneMuted}
setShowInspector={onChangeShowInspector}
showInspector={showInspector}
roomId={roomId}
/>
);

12
src/room/InCallView.jsx
View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useCallback, useMemo } from "react";
import React, { useCallback, useMemo, useRef } from "react";
import styles from "./InCallView.module.css";
import {
HangupButton,
@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { usePreventScroll } from "@react-aria/overlays";
import { useMediaHandler } from "../settings/useMediaHandler";
import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@ -57,14 +58,16 @@ export function InCallView({
toggleScreensharing,
isScreensharing,
screenshareFeeds,
setShowInspector,
showInspector,
roomId,
}) {
usePreventScroll();
const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
const { audioOutput } = useMediaHandler();
const [showInspector] = useShowInspector();
const audioContext = useRef();
if (!audioContext.current) audioContext.current = new AudioContext();
const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
useModalTriggerState();
@ -151,6 +154,7 @@ export function InCallView({
getAvatar={renderAvatar}
showName={items.length > 2 || item.focused}
audioOutputDevice={audioOutput}
audioContext={audioContext.current}
disableSpeakingIndicator={items.length < 3}
{...rest}
/>
@ -169,8 +173,6 @@ export function InCallView({
<OverflowMenu
inCall
roomId={roomId}
setShowInspector={setShowInspector}
showInspector={showInspector}
client={client}
groupCall={groupCall}
showInvite={true}

4
src/room/LobbyView.jsx
View file

@ -41,8 +41,6 @@ export function LobbyView({
localVideoMuted,
toggleLocalVideoMuted,
toggleMicrophoneMuted,
setShowInspector,
showInspector,
roomId,
}) {
const { stream } = useCallFeed(localCallFeed);
@ -101,8 +99,6 @@ export function LobbyView({
localVideoMuted={localVideoMuted}
toggleLocalVideoMuted={toggleLocalVideoMuted}
toggleMicrophoneMuted={toggleMicrophoneMuted}
setShowInspector={setShowInspector}
showInspector={showInspector}
stream={stream}
audioOutput={audioOutput}
/>

10
src/room/OverflowMenu.jsx
View file

@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
export function OverflowMenu({
roomId,
setShowInspector,
showInspector,
inCall,
groupCall,
showInvite,
@ -88,13 +86,7 @@ export function OverflowMenu({
</Menu>
)}
</PopoverMenuTrigger>
{settingsModalState.isOpen && (
<SettingsModal
{...settingsModalProps}
setShowInspector={setShowInspector}
showInspector={showInspector}
/>
)}
{settingsModalState.isOpen && <SettingsModal {...settingsModalProps} />}
{inviteModalState.isOpen && (
<InviteModal roomId={roomId} {...inviteModalProps} />
)}

6
src/room/PTTButton.module.css
View file

@ -9,17 +9,15 @@
background-color: #21262c;
position: relative;
padding: 0;
cursor: pointer;
}
.talking {
background-color: #0dbd8b;
box-shadow: 0px 0px 0px 17px rgba(13, 189, 139, 0.2),
0px 0px 0px 34px rgba(13, 189, 139, 0.2);
cursor: unset;
}
.error {
background-color: #ff5b55;
border-color: #ff5b55;
box-shadow: 0px 0px 0px 17px rgba(255, 91, 85, 0.2),
0px 0px 0px 34px rgba(255, 91, 85, 0.2);
}

27
src/room/PTTButton.tsx
View file

@ -16,6 +16,7 @@ limitations under the License.
import React, { useCallback, useEffect, useState, createRef } from "react";
import classNames from "classnames";
import { useSpring, animated } from "@react-spring/web";
import styles from "./PTTButton.module.css";
import { ReactComponent as MicIcon } from "../icons/Mic.svg";
@ -27,6 +28,7 @@ interface Props {
activeSpeakerDisplayName: string;
activeSpeakerAvatarUrl: string;
activeSpeakerIsLocalUser: boolean;
activeSpeakerVolume: number;
size: number;
startTalking: () => void;
stopTalking: () => void;
@ -44,6 +46,7 @@ export const PTTButton: React.FC<Props> = ({
activeSpeakerDisplayName,
activeSpeakerAvatarUrl,
activeSpeakerIsLocalUser,
activeSpeakerVolume,
size,
startTalking,
stopTalking,
@ -130,12 +133,32 @@ export const PTTButton: React.FC<Props> = ({
);
};
}, [onWindowMouseUp, onWindowTouchEnd, onButtonTouchStart, buttonRef]);
const { shadow } = useSpring({
shadow: (Math.max(activeSpeakerVolume, -70) + 70) * 0.6,
config: {
clamp: true,
tension: 300,
},
});
const shadowColor = showTalkOverError
? "var(--alert-20)"
: "var(--primaryColor-20)";
return (
<button
<animated.button
className={classNames(styles.pttButton, {
[styles.talking]: activeSpeakerUserId,
[styles.error]: showTalkOverError,
})}
style={{
boxShadow: shadow.to(
(s) =>
`0px 0px 0px ${s}px ${shadowColor}, 0px 0px 0px ${
2 * s
}px ${shadowColor}`
),
}}
onMouseDown={onButtonMouseDown}
ref={buttonRef}
>
@ -154,6 +177,6 @@ export const PTTButton: React.FC<Props> = ({
className={styles.avatar}
/>
)}
</button>
</animated.button>
);
};

17
src/room/PTTCallView.tsx
View file

@ -44,8 +44,11 @@ function getPromptText(
activeSpeakerIsLocalUser: boolean,
talkOverEnabled: boolean,
activeSpeakerUserId: string,
activeSpeakerDisplayName: string
activeSpeakerDisplayName: string,
connected: boolean
): string {
if (!connected) return "Connection Lost";
const isTouchScreen = Boolean(window.ontouchstart !== undefined);
if (showTalkOverError) {
@ -84,8 +87,6 @@ interface Props {
participants: RoomMember[];
userMediaFeeds: CallFeed[];
onLeave: () => void;
setShowInspector: (boolean) => void;
showInspector: boolean;
}
export const PTTCallView: React.FC<Props> = ({
@ -97,8 +98,6 @@ export const PTTCallView: React.FC<Props> = ({
participants,
userMediaFeeds,
onLeave,
setShowInspector,
showInspector,
}) => {
const { modalState: inviteModalState, modalProps: inviteModalProps } =
useModalTriggerState();
@ -124,9 +123,11 @@ export const PTTCallView: React.FC<Props> = ({
talkOverEnabled,
setTalkOverEnabled,
activeSpeakerUserId,
activeSpeakerVolume,
startTalking,
stopTalking,
transmitBlocked,
connected,
} = usePTT(
client,
groupCall,
@ -189,8 +190,6 @@ export const PTTCallView: React.FC<Props> = ({
<OverflowMenu
inCall
roomId={roomId}
setShowInspector={setShowInspector}
showInspector={showInspector}
client={client}
groupCall={groupCall}
showInvite={false}
@ -223,6 +222,7 @@ export const PTTCallView: React.FC<Props> = ({
activeSpeakerDisplayName={activeSpeakerDisplayName}
activeSpeakerAvatarUrl={activeSpeakerAvatarUrl}
activeSpeakerIsLocalUser={activeSpeakerIsLocalUser}
activeSpeakerVolume={activeSpeakerVolume}
size={pttButtonSize}
startTalking={startTalking}
stopTalking={stopTalking}
@ -234,7 +234,8 @@ export const PTTCallView: React.FC<Props> = ({
activeSpeakerIsLocalUser,
talkOverEnabled,
activeSpeakerUserId,
activeSpeakerDisplayName
activeSpeakerDisplayName,
connected
)}
</p>
{userMediaFeeds.map((callFeed) => (

4
src/room/VideoPreview.jsx
View file

@ -35,8 +35,6 @@ export function VideoPreview({
localVideoMuted,
toggleLocalVideoMuted,
toggleMicrophoneMuted,
setShowInspector,
showInspector,
audioOutput,
stream,
}) {
@ -83,8 +81,6 @@ export function VideoPreview({
/>
<OverflowMenu
roomId={roomId}
setShowInspector={setShowInspector}
showInspector={showInspector}
client={client}
feedbackModalState={feedbackModalState}
feedbackModalProps={feedbackModalProps}

83
src/room/useGroupCall.js → src/room/useGroupCall.ts
View file

@ -18,10 +18,57 @@ import { useCallback, useEffect, useState } from "react";
import {
GroupCallEvent,
GroupCallState,
GroupCall,
} from "matrix-js-sdk/src/webrtc/groupCall";
import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { usePageUnload } from "./usePageUnload";
export function useGroupCall(groupCall) {
export interface UseGroupCallType {
state: GroupCallState;
calls: MatrixCall[];
localCallFeed: CallFeed;
activeSpeaker: string;
userMediaFeeds: CallFeed[];
microphoneMuted: boolean;
localVideoMuted: boolean;
error: Error;
initLocalCallFeed: () => void;
enter: () => void;
leave: () => void;
toggleLocalVideoMuted: () => void;
toggleMicrophoneMuted: () => void;
toggleScreensharing: () => void;
requestingScreenshare: boolean;
isScreensharing: boolean;
screenshareFeeds: CallFeed[];
localScreenshareFeed: CallFeed;
localDesktopCapturerSourceId: string;
participants: RoomMember[];
hasLocalParticipant: boolean;
}
interface State {
state: GroupCallState;
calls: MatrixCall[];
localCallFeed: CallFeed;
activeSpeaker: string;
userMediaFeeds: CallFeed[];
error: Error;
microphoneMuted: boolean;
localVideoMuted: boolean;
screenshareFeeds: CallFeed[];
localScreenshareFeed: CallFeed;
localDesktopCapturerSourceId: string;
isScreensharing: boolean;
requestingScreenshare: boolean;
participants: RoomMember[];
hasLocalParticipant: boolean;
}
export function useGroupCall(groupCall: GroupCall): UseGroupCallType {
const [
{
state,
@ -41,20 +88,25 @@ export function useGroupCall(groupCall) {
requestingScreenshare,
},
setState,
] = useState({
] = useState<State>({
state: GroupCallState.LocalCallFeedUninitialized,
calls: [],
localCallFeed: null,
activeSpeaker: null,
userMediaFeeds: [],
error: null,
microphoneMuted: false,
localVideoMuted: false,
screenshareFeeds: [],
isScreensharing: false,
screenshareFeeds: [],
localScreenshareFeed: null,
localDesktopCapturerSourceId: null,
requestingScreenshare: false,
participants: [],
hasLocalParticipant: false,
});
const updateState = (state) =>
const updateState = (state: Partial<State>) =>
setState((prevState) => ({ ...prevState, ...state }));
useEffect(() => {
@ -75,25 +127,28 @@ export function useGroupCall(groupCall) {
});
}
function onUserMediaFeedsChanged(userMediaFeeds) {
function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void {
updateState({
userMediaFeeds: [...userMediaFeeds],
});
}
function onScreenshareFeedsChanged(screenshareFeeds) {
function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void {
updateState({
screenshareFeeds: [...screenshareFeeds],
});
}
function onActiveSpeakerChanged(activeSpeaker) {
function onActiveSpeakerChanged(activeSpeaker: string): void {
updateState({
activeSpeaker: activeSpeaker,
});
}
function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) {
function onLocalMuteStateChanged(
microphoneMuted: boolean,
localVideoMuted: boolean
): void {
updateState({
microphoneMuted,
localVideoMuted,
@ -101,10 +156,10 @@ export function useGroupCall(groupCall) {
}
function onLocalScreenshareStateChanged(
isScreensharing,
localScreenshareFeed,
localDesktopCapturerSourceId
) {
isScreensharing: boolean,
localScreenshareFeed: CallFeed,
localDesktopCapturerSourceId: string
): void {
updateState({
isScreensharing,
localScreenshareFeed,
@ -112,13 +167,13 @@ export function useGroupCall(groupCall) {
});
}
function onCallsChanged(calls) {
function onCallsChanged(calls: MatrixCall[]): void {
updateState({
calls: [...calls],
});
}
function onParticipantsChanged(participants) {
function onParticipantsChanged(participants: RoomMember[]): void {
updateState({
participants: [...participants],
hasLocalParticipant: groupCall.hasLocalParticipant(),

85
src/room/usePTT.ts
View file

@ -15,10 +15,11 @@ limitations under the License.
*/
import { useCallback, useEffect, useState } from "react";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
import { logger } from "matrix-js-sdk/src/logger";
import { SyncState } from "matrix-js-sdk/src/sync";
import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
@ -30,6 +31,21 @@ function getActiveSpeakerFeed(
): CallFeed | null {
const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
// make sure the feeds are in a deterministic order so every client picks
// the same one as the active speaker. The custom sort function sorts
// by user ID, so needs a collator of some kind to compare. We make a
// specific one to help ensure every client sorts the same way
// although of course user IDs shouldn't contain accented characters etc.
// anyway).
const collator = new Intl.Collator("en", {
sensitivity: "variant",
usage: "sort",
ignorePunctuation: false,
});
activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number =>
collator.compare(a.userId, b.userId)
);
let activeSpeakerFeed = null;
let highestPowerLevel = null;
for (const feed of activeSpeakerFeeds) {
@ -49,9 +65,15 @@ export interface PTTState {
talkOverEnabled: boolean;
setTalkOverEnabled: (boolean) => void;
activeSpeakerUserId: string;
activeSpeakerVolume: number;
startTalking: () => void;
stopTalking: () => void;
transmitBlocked: boolean;
// connected is actually an indication of whether we're connected to the HS
// (ie. the client's syncing state) rather than media connection, since
// it's peer to peer so we can't really say which peer is 'disconnected' if
// there's only one other person in the call and they've lost Internet.
connected: boolean;
}
export const usePTT = (
@ -87,6 +109,7 @@ export const usePTT = (
isAdmin,
talkOverEnabled,
activeSpeakerUserId,
activeSpeakerVolume,
transmitBlocked,
},
setState,
@ -100,6 +123,7 @@ export const usePTT = (
talkOverEnabled: false,
pttButtonHeld: false,
activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
activeSpeakerVolume: -Infinity,
transmitBlocked: false,
};
});
@ -131,15 +155,11 @@ export const usePTT = (
playClip(PTTClipID.BLOCKED);
}
setState((prevState) => {
return {
setState((prevState) => ({
...prevState,
activeSpeakerUserId: activeSpeakerFeed
? activeSpeakerFeed.userId
: null,
activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
transmitBlocked: blocked,
};
});
}));
}, [
playClip,
groupCall,
@ -152,7 +172,7 @@ export const usePTT = (
useEffect(() => {
for (const callFeed of userMediaFeeds) {
callFeed.addListener(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
}
const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
@ -164,14 +184,30 @@ export const usePTT = (
return () => {
for (const callFeed of userMediaFeeds) {
callFeed.removeListener(
CallFeedEvent.MuteStateChanged,
onMuteStateChanged
);
callFeed.off(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
}
};
}, [userMediaFeeds, onMuteStateChanged, groupCall]);
const onVolumeChanged = useCallback((volume: number) => {
setState((prevState) => ({
...prevState,
activeSpeakerVolume: volume,
}));
}, []);
useEffect(() => {
const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
activeSpeakerFeed?.on(CallFeedEvent.VolumeChanged, onVolumeChanged);
return () => {
activeSpeakerFeed?.off(CallFeedEvent.VolumeChanged, onVolumeChanged);
setState((prevState) => ({
...prevState,
activeSpeakerVolume: -Infinity,
}));
};
}, [activeSpeakerUserId, onVolumeChanged, userMediaFeeds, groupCall]);
const startTalking = useCallback(async () => {
if (pttButtonHeld) return;
@ -211,6 +247,17 @@ export const usePTT = (
setMicMuteWrapper(true);
}, [setMicMuteWrapper]);
// separate state for connected: we set it separately from other things
// in the client sync callback
const [connected, setConnected] = useState(true);
const onClientSync = useCallback(
(syncState: SyncState) => {
setConnected(syncState !== SyncState.Error);
},
[setConnected]
);
useEffect(() => {
function onKeyDown(event: KeyboardEvent): void {
if (event.code === "Space") {
@ -260,8 +307,18 @@ export const usePTT = (
pttButtonHeld,
enablePTTButton,
setMicMuteWrapper,
client,
onClientSync,
]);
useEffect(() => {
client.on(ClientEvent.Sync, onClientSync);
return () => {
client.removeListener(ClientEvent.Sync, onClientSync);
};
}, [client, onClientSync]);
const setTalkOverEnabled = useCallback((talkOverEnabled) => {
setState((prevState) => ({
...prevState,
@ -275,8 +332,10 @@ export const usePTT = (
talkOverEnabled,
setTalkOverEnabled,
activeSpeakerUserId,
activeSpeakerVolume,
startTalking,
stopTalking,
transmitBlocked,
connected,
};
};

18
src/settings/SettingsModal.jsx
View file

@ -24,12 +24,13 @@ import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
import { SelectInput } from "../input/SelectInput";
import { Item } from "@react-stately/collections";
import { useMediaHandler } from "./useMediaHandler";
import { useSpatialAudio, useShowInspector } from "./useSetting";
import { FieldRow, InputField } from "../input/Input";
import { Button } from "../button";
import { useDownloadDebugLog } from "./submit-rageshake";
import { Body } from "../typography/Typography";
export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
export const SettingsModal = (props) => {
const {
audioInput,
audioInputs,
@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
audioOutputs,
setAudioOutput,
} = useMediaHandler();
const [spatialAudio, setSpatialAudio] = useSpatialAudio();
const [showInspector, setShowInspector] = useShowInspector();
const downloadDebugLog = useDownloadDebugLog();
@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
isDismissable
mobileFullScreen
className={styles.settingsModal}
{...rest}
{...props}
>
<TabContainer className={styles.tabContainer}>
<TabItem
@ -81,6 +84,15 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
))}
</SelectInput>
)}
<FieldRow>
<InputField
id="spatialAudio"
label="Spatial audio (experimental)"
type="checkbox"
checked={spatialAudio}
onChange={(e) => setSpatialAudio(e.target.checked)}
/>
</FieldRow>
</TabItem>
<TabItem
title={
@ -130,4 +142,4 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
</TabContainer>
</Modal>
);
}
};

56
src/settings/useSetting.ts Normal file
View file

@ -0,0 +1,56 @@
/*
Copyright 2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { EventEmitter } from "events";
import { useMemo, useState, useEffect, useCallback } from "react";
// Bus to notify other useSetting consumers when a setting is changed
const settingsBus = new EventEmitter();
// Like useState, but reads from and persists the value to localStorage
const useSetting = <T>(
name: string,
defaultValue: T
): [T, (value: T) => void] => {
const key = useMemo(() => `matrix-setting-${name}`, [name]);
const [value, setValue] = useState<T>(() => {
const item = localStorage.getItem(key);
return item == null ? defaultValue : JSON.parse(item);
});
useEffect(() => {
settingsBus.on(name, setValue);
return () => {
settingsBus.off(name, setValue);
};
}, [name, setValue]);
return [
value,
useCallback(
(newValue: T) => {
setValue(newValue);
localStorage.setItem(key, JSON.stringify(newValue));
settingsBus.emit(name, newValue);
},
[name, key, setValue]
),
];
};
export const useSpatialAudio = () => useSetting("spatial-audio", false);
export const useShowInspector = () => useSetting("show-inspector", false);

14
src/video-grid/VideoTile.jsx
View file

@ -14,14 +14,16 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import React, { forwardRef } from "react";
import { animated } from "@react-spring/web";
import classNames from "classnames";
import styles from "./VideoTile.module.css";
import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
export function VideoTile({
export const VideoTile = forwardRef(
(
{
className,
isLocal,
speaking,
@ -34,7 +36,9 @@ export function VideoTile({
showName,
mediaRef,
...rest
}) {
},
ref
) => {
return (
<animated.div
className={classNames(styles.videoTile, className, {
@ -43,6 +47,7 @@ export function VideoTile({
[styles.muted]: audioMuted,
[styles.screenshare]: screenshare,
})}
ref={ref}
{...rest}
>
{(videoMuted || noVideo) && (
@ -67,4 +72,5 @@ export function VideoTile({
<video ref={mediaRef} playsInline disablePictureInPicture />
</animated.div>
);
}
}
);

4
src/video-grid/VideoTile.module.css
View file

@ -5,6 +5,10 @@
overflow: hidden;
cursor: pointer;
touch-action: none;
/* HACK: This has no visual effect due to the short duration, but allows the
JS to detect movement via the transform property for audio spatialization */
transition: transform 0.000000001s;
}
.videoTile * {

11
src/video-grid/VideoTileContainer.jsx
View file

@ -17,7 +17,7 @@ limitations under the License.
import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
import React from "react";
import { useCallFeed } from "./useCallFeed";
import { useMediaStream } from "./useMediaStream";
import { useSpatialMediaStream } from "./useMediaStream";
import { useRoomMemberName } from "./useRoomMemberName";
import { VideoTile } from "./VideoTile";
@ -28,6 +28,7 @@ export function VideoTileContainer({
getAvatar,
showName,
audioOutputDevice,
audioContext,
disableSpeakingIndicator,
...rest
}) {
@ -42,7 +43,12 @@ export function VideoTileContainer({
member,
} = useCallFeed(item.callFeed);
const { rawDisplayName } = useRoomMemberName(member);
const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
const [tileRef, mediaRef] = useSpatialMediaStream(
stream,
audioOutputDevice,
audioContext,
isLocal
);
// Firefox doesn't respect the disablePictureInPicture attribute
// https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@ -57,6 +63,7 @@ export function VideoTileContainer({
screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
name={rawDisplayName}
showName={showName}
ref={tileRef}
mediaRef={mediaRef}
avatar={getAvatar && getAvatar(member, width, height)}
{...rest}

71
src/video-grid/useMediaStream.js
View file

@ -16,6 +16,8 @@ limitations under the License.
import { useRef, useEffect } from "react";
import { useSpatialAudio } from "../settings/useSetting";
export function useMediaStream(stream, audioOutputDevice, mute = false) {
const mediaRef = useRef();
@ -55,7 +57,8 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
mediaRef.current !== undefined
) {
console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
mediaRef.current.setSinkId(audioOutputDevice);
// Chrome for Android doesn't support this
mediaRef.current.setSinkId?.(audioOutputDevice);
}
}, [audioOutputDevice]);
@ -73,3 +76,69 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
return mediaRef;
}
export const useSpatialMediaStream = (
stream,
audioOutputDevice,
audioContext,
mute = false
) => {
const tileRef = useRef();
const [spatialAudio] = useSpatialAudio();
// If spatial audio is enabled, we handle audio separately from the video element
const mediaRef = useMediaStream(
stream,
audioOutputDevice,
spatialAudio || mute
);
const pannerNodeRef = useRef();
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
refDistance: 3,
});
}
const sourceRef = useRef();
useEffect(() => {
if (spatialAudio && tileRef.current && !mute) {
if (!sourceRef.current) {
sourceRef.current = audioContext.createMediaStreamSource(stream);
}
const tile = tileRef.current;
const source = sourceRef.current;
const pannerNode = pannerNodeRef.current;
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
// Position the source relative to its placement in the window
pannerNodeRef.current.positionX.value =
(bounds.x + bounds.width / 2) / windowSize - 0.5;
pannerNodeRef.current.positionY.value =
(bounds.y + bounds.height / 2) / windowSize - 0.5;
// Put the source in front of the listener
pannerNodeRef.current.positionZ.value = -2;
};
updatePosition();
source.connect(pannerNode);
pannerNode.connect(audioContext.destination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
// moves, because useMeasure, IntersectionObserver, etc. all have no
// ability to track changes in the CSS transform property
tile.addEventListener("transitionrun", updatePosition);
return () => {
tile.removeEventListener("transitionrun", updatePosition);
source.disconnect();
pannerNode.disconnect();
};
}
}, [stream, spatialAudio, audioContext, mute]);
return [tileRef, mediaRef];
};