diff --git a/src/index.css b/src/index.css
index 25562e3..0272651 100644
--- a/src/index.css
+++ b/src/index.css
@@ -26,6 +26,8 @@ limitations under the License.
--inter-unicode-range: U+0000-20e2, U+20e4-23ce, U+23d0-24c1, U+24c3-259f,
U+25c2-2664, U+2666-2763, U+2765-2b05, U+2b07-2b1b, U+2b1d-10FFFF;
--primaryColor: #0dbd8b;
+ --primaryColor-20: #0dbd8b33;
+ --alert-20: #ff5b5533;
--bgColor1: #15191e;
--bgColor2: #21262c;
--bgColor3: #444;
diff --git a/src/room/GroupCallView.jsx b/src/room/GroupCallView.jsx
index dcd9da3..2809b82 100644
--- a/src/room/GroupCallView.jsx
+++ b/src/room/GroupCallView.jsx
@@ -33,19 +33,6 @@ export function GroupCallView({
roomId,
groupCall,
}) {
- const [showInspector, setShowInspector] = useState(
- () => !!localStorage.getItem("matrix-group-call-inspector")
- );
- const onChangeShowInspector = useCallback((show) => {
- setShowInspector(show);
-
- if (show) {
- localStorage.setItem("matrix-group-call-inspector", "true");
- } else {
- localStorage.removeItem("matrix-group-call-inspector");
- }
- }, []);
-
const {
state,
error,
@@ -104,8 +91,6 @@ export function GroupCallView({
participants={participants}
userMediaFeeds={userMediaFeeds}
onLeave={onLeave}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
/>
);
} else {
@@ -126,8 +111,6 @@ export function GroupCallView({
isScreensharing={isScreensharing}
localScreenshareFeed={localScreenshareFeed}
screenshareFeeds={screenshareFeeds}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
roomId={roomId}
/>
);
@@ -156,8 +139,6 @@ export function GroupCallView({
localVideoMuted={localVideoMuted}
toggleLocalVideoMuted={toggleLocalVideoMuted}
toggleMicrophoneMuted={toggleMicrophoneMuted}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
roomId={roomId}
/>
);
diff --git a/src/room/InCallView.jsx b/src/room/InCallView.jsx
index a474963..2c6240e 100644
--- a/src/room/InCallView.jsx
+++ b/src/room/InCallView.jsx
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-import React, { useCallback, useMemo } from "react";
+import React, { useCallback, useMemo, useRef } from "react";
import styles from "./InCallView.module.css";
import {
HangupButton,
@@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { usePreventScroll } from "@react-aria/overlays";
import { useMediaHandler } from "../settings/useMediaHandler";
+import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@@ -57,14 +58,16 @@ export function InCallView({
toggleScreensharing,
isScreensharing,
screenshareFeeds,
- setShowInspector,
- showInspector,
roomId,
}) {
usePreventScroll();
const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
const { audioOutput } = useMediaHandler();
+ const [showInspector] = useShowInspector();
+
+ const audioContext = useRef();
+ if (!audioContext.current) audioContext.current = new AudioContext();
const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
useModalTriggerState();
@@ -151,6 +154,7 @@ export function InCallView({
getAvatar={renderAvatar}
showName={items.length > 2 || item.focused}
audioOutputDevice={audioOutput}
+ audioContext={audioContext.current}
disableSpeakingIndicator={items.length < 3}
{...rest}
/>
@@ -169,8 +173,6 @@ export function InCallView({
diff --git a/src/room/OverflowMenu.jsx b/src/room/OverflowMenu.jsx
index 281995c..c5810f0 100644
--- a/src/room/OverflowMenu.jsx
+++ b/src/room/OverflowMenu.jsx
@@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
export function OverflowMenu({
roomId,
- setShowInspector,
- showInspector,
inCall,
groupCall,
showInvite,
@@ -88,13 +86,7 @@ export function OverflowMenu({
)}
- {settingsModalState.isOpen && (
-
- )}
+ {settingsModalState.isOpen && }
{inviteModalState.isOpen && (
)}
diff --git a/src/room/PTTButton.module.css b/src/room/PTTButton.module.css
index 1a8dd17..fb849ad 100644
--- a/src/room/PTTButton.module.css
+++ b/src/room/PTTButton.module.css
@@ -9,17 +9,15 @@
background-color: #21262c;
position: relative;
padding: 0;
+ cursor: pointer;
}
.talking {
background-color: #0dbd8b;
- box-shadow: 0px 0px 0px 17px rgba(13, 189, 139, 0.2),
- 0px 0px 0px 34px rgba(13, 189, 139, 0.2);
+ cursor: unset;
}
.error {
background-color: #ff5b55;
border-color: #ff5b55;
- box-shadow: 0px 0px 0px 17px rgba(255, 91, 85, 0.2),
- 0px 0px 0px 34px rgba(255, 91, 85, 0.2);
}
diff --git a/src/room/PTTButton.tsx b/src/room/PTTButton.tsx
index 4da324c..62299a1 100644
--- a/src/room/PTTButton.tsx
+++ b/src/room/PTTButton.tsx
@@ -16,6 +16,7 @@ limitations under the License.
import React, { useCallback, useEffect, useState, createRef } from "react";
import classNames from "classnames";
+import { useSpring, animated } from "@react-spring/web";
import styles from "./PTTButton.module.css";
import { ReactComponent as MicIcon } from "../icons/Mic.svg";
@@ -27,6 +28,7 @@ interface Props {
activeSpeakerDisplayName: string;
activeSpeakerAvatarUrl: string;
activeSpeakerIsLocalUser: boolean;
+ activeSpeakerVolume: number;
size: number;
startTalking: () => void;
stopTalking: () => void;
@@ -44,6 +46,7 @@ export const PTTButton: React.FC = ({
activeSpeakerDisplayName,
activeSpeakerAvatarUrl,
activeSpeakerIsLocalUser,
+ activeSpeakerVolume,
size,
startTalking,
stopTalking,
@@ -130,12 +133,32 @@ export const PTTButton: React.FC = ({
);
};
}, [onWindowMouseUp, onWindowTouchEnd, onButtonTouchStart, buttonRef]);
+
+ const { shadow } = useSpring({
+ shadow: (Math.max(activeSpeakerVolume, -70) + 70) * 0.6,
+ config: {
+ clamp: true,
+ tension: 300,
+ },
+ });
+ const shadowColor = showTalkOverError
+ ? "var(--alert-20)"
+ : "var(--primaryColor-20)";
+
return (
-
+
);
};
diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx
index 4b395f8..5e3f304 100644
--- a/src/room/PTTCallView.tsx
+++ b/src/room/PTTCallView.tsx
@@ -44,8 +44,11 @@ function getPromptText(
activeSpeakerIsLocalUser: boolean,
talkOverEnabled: boolean,
activeSpeakerUserId: string,
- activeSpeakerDisplayName: string
+ activeSpeakerDisplayName: string,
+ connected: boolean
): string {
+ if (!connected) return "Connection Lost";
+
const isTouchScreen = Boolean(window.ontouchstart !== undefined);
if (showTalkOverError) {
@@ -84,8 +87,6 @@ interface Props {
participants: RoomMember[];
userMediaFeeds: CallFeed[];
onLeave: () => void;
- setShowInspector: (boolean) => void;
- showInspector: boolean;
}
export const PTTCallView: React.FC = ({
@@ -97,8 +98,6 @@ export const PTTCallView: React.FC = ({
participants,
userMediaFeeds,
onLeave,
- setShowInspector,
- showInspector,
}) => {
const { modalState: inviteModalState, modalProps: inviteModalProps } =
useModalTriggerState();
@@ -124,9 +123,11 @@ export const PTTCallView: React.FC = ({
talkOverEnabled,
setTalkOverEnabled,
activeSpeakerUserId,
+ activeSpeakerVolume,
startTalking,
stopTalking,
transmitBlocked,
+ connected,
} = usePTT(
client,
groupCall,
@@ -189,8 +190,6 @@ export const PTTCallView: React.FC = ({
= ({
activeSpeakerDisplayName={activeSpeakerDisplayName}
activeSpeakerAvatarUrl={activeSpeakerAvatarUrl}
activeSpeakerIsLocalUser={activeSpeakerIsLocalUser}
+ activeSpeakerVolume={activeSpeakerVolume}
size={pttButtonSize}
startTalking={startTalking}
stopTalking={stopTalking}
@@ -234,7 +234,8 @@ export const PTTCallView: React.FC = ({
activeSpeakerIsLocalUser,
talkOverEnabled,
activeSpeakerUserId,
- activeSpeakerDisplayName
+ activeSpeakerDisplayName,
+ connected
)}
{userMediaFeeds.map((callFeed) => (
diff --git a/src/room/VideoPreview.jsx b/src/room/VideoPreview.jsx
index d0e1774..996e4ae 100644
--- a/src/room/VideoPreview.jsx
+++ b/src/room/VideoPreview.jsx
@@ -35,8 +35,6 @@ export function VideoPreview({
localVideoMuted,
toggleLocalVideoMuted,
toggleMicrophoneMuted,
- setShowInspector,
- showInspector,
audioOutput,
stream,
}) {
@@ -83,8 +81,6 @@ export function VideoPreview({
/>
void;
+ enter: () => void;
+ leave: () => void;
+ toggleLocalVideoMuted: () => void;
+ toggleMicrophoneMuted: () => void;
+ toggleScreensharing: () => void;
+ requestingScreenshare: boolean;
+ isScreensharing: boolean;
+ screenshareFeeds: CallFeed[];
+ localScreenshareFeed: CallFeed;
+ localDesktopCapturerSourceId: string;
+ participants: RoomMember[];
+ hasLocalParticipant: boolean;
+}
+
+interface State {
+ state: GroupCallState;
+ calls: MatrixCall[];
+ localCallFeed: CallFeed;
+ activeSpeaker: string;
+ userMediaFeeds: CallFeed[];
+ error: Error;
+ microphoneMuted: boolean;
+ localVideoMuted: boolean;
+ screenshareFeeds: CallFeed[];
+ localScreenshareFeed: CallFeed;
+ localDesktopCapturerSourceId: string;
+ isScreensharing: boolean;
+ requestingScreenshare: boolean;
+ participants: RoomMember[];
+ hasLocalParticipant: boolean;
+}
+
+export function useGroupCall(groupCall: GroupCall): UseGroupCallType {
const [
{
state,
@@ -41,20 +88,25 @@ export function useGroupCall(groupCall) {
requestingScreenshare,
},
setState,
- ] = useState({
+ ] = useState({
state: GroupCallState.LocalCallFeedUninitialized,
calls: [],
+ localCallFeed: null,
+ activeSpeaker: null,
userMediaFeeds: [],
+ error: null,
microphoneMuted: false,
localVideoMuted: false,
- screenshareFeeds: [],
isScreensharing: false,
+ screenshareFeeds: [],
+ localScreenshareFeed: null,
+ localDesktopCapturerSourceId: null,
requestingScreenshare: false,
participants: [],
hasLocalParticipant: false,
});
- const updateState = (state) =>
+ const updateState = (state: Partial) =>
setState((prevState) => ({ ...prevState, ...state }));
useEffect(() => {
@@ -75,25 +127,28 @@ export function useGroupCall(groupCall) {
});
}
- function onUserMediaFeedsChanged(userMediaFeeds) {
+ function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void {
updateState({
userMediaFeeds: [...userMediaFeeds],
});
}
- function onScreenshareFeedsChanged(screenshareFeeds) {
+ function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void {
updateState({
screenshareFeeds: [...screenshareFeeds],
});
}
- function onActiveSpeakerChanged(activeSpeaker) {
+ function onActiveSpeakerChanged(activeSpeaker: string): void {
updateState({
activeSpeaker: activeSpeaker,
});
}
- function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) {
+ function onLocalMuteStateChanged(
+ microphoneMuted: boolean,
+ localVideoMuted: boolean
+ ): void {
updateState({
microphoneMuted,
localVideoMuted,
@@ -101,10 +156,10 @@ export function useGroupCall(groupCall) {
}
function onLocalScreenshareStateChanged(
- isScreensharing,
- localScreenshareFeed,
- localDesktopCapturerSourceId
- ) {
+ isScreensharing: boolean,
+ localScreenshareFeed: CallFeed,
+ localDesktopCapturerSourceId: string
+ ): void {
updateState({
isScreensharing,
localScreenshareFeed,
@@ -112,13 +167,13 @@ export function useGroupCall(groupCall) {
});
}
- function onCallsChanged(calls) {
+ function onCallsChanged(calls: MatrixCall[]): void {
updateState({
calls: [...calls],
});
}
- function onParticipantsChanged(participants) {
+ function onParticipantsChanged(participants: RoomMember[]): void {
updateState({
participants: [...participants],
hasLocalParticipant: groupCall.hasLocalParticipant(),
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index 5965c54..7710655 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -15,10 +15,11 @@ limitations under the License.
*/
import { useCallback, useEffect, useState } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
import { logger } from "matrix-js-sdk/src/logger";
+import { SyncState } from "matrix-js-sdk/src/sync";
import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
@@ -30,6 +31,21 @@ function getActiveSpeakerFeed(
): CallFeed | null {
const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
+ // make sure the feeds are in a deterministic order so every client picks
+ // the same one as the active speaker. The custom sort function sorts
+ // by user ID, so needs a collator of some kind to compare. We make a
+ // specific one to help ensure every client sorts the same way
+ // although of course user IDs shouldn't contain accented characters etc.
+ // anyway).
+ const collator = new Intl.Collator("en", {
+ sensitivity: "variant",
+ usage: "sort",
+ ignorePunctuation: false,
+ });
+ activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number =>
+ collator.compare(a.userId, b.userId)
+ );
+
let activeSpeakerFeed = null;
let highestPowerLevel = null;
for (const feed of activeSpeakerFeeds) {
@@ -49,9 +65,15 @@ export interface PTTState {
talkOverEnabled: boolean;
setTalkOverEnabled: (boolean) => void;
activeSpeakerUserId: string;
+ activeSpeakerVolume: number;
startTalking: () => void;
stopTalking: () => void;
transmitBlocked: boolean;
+ // connected is actually an indication of whether we're connected to the HS
+ // (ie. the client's syncing state) rather than media connection, since
+ // it's peer to peer so we can't really say which peer is 'disconnected' if
+ // there's only one other person in the call and they've lost Internet.
+ connected: boolean;
}
export const usePTT = (
@@ -87,6 +109,7 @@ export const usePTT = (
isAdmin,
talkOverEnabled,
activeSpeakerUserId,
+ activeSpeakerVolume,
transmitBlocked,
},
setState,
@@ -100,6 +123,7 @@ export const usePTT = (
talkOverEnabled: false,
pttButtonHeld: false,
activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
+ activeSpeakerVolume: -Infinity,
transmitBlocked: false,
};
});
@@ -131,15 +155,11 @@ export const usePTT = (
playClip(PTTClipID.BLOCKED);
}
- setState((prevState) => {
- return {
- ...prevState,
- activeSpeakerUserId: activeSpeakerFeed
- ? activeSpeakerFeed.userId
- : null,
- transmitBlocked: blocked,
- };
- });
+ setState((prevState) => ({
+ ...prevState,
+ activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
+ transmitBlocked: blocked,
+ }));
}, [
playClip,
groupCall,
@@ -152,7 +172,7 @@ export const usePTT = (
useEffect(() => {
for (const callFeed of userMediaFeeds) {
- callFeed.addListener(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
+ callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
}
const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
@@ -164,14 +184,30 @@ export const usePTT = (
return () => {
for (const callFeed of userMediaFeeds) {
- callFeed.removeListener(
- CallFeedEvent.MuteStateChanged,
- onMuteStateChanged
- );
+ callFeed.off(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
}
};
}, [userMediaFeeds, onMuteStateChanged, groupCall]);
+ const onVolumeChanged = useCallback((volume: number) => {
+ setState((prevState) => ({
+ ...prevState,
+ activeSpeakerVolume: volume,
+ }));
+ }, []);
+
+ useEffect(() => {
+ const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
+ activeSpeakerFeed?.on(CallFeedEvent.VolumeChanged, onVolumeChanged);
+ return () => {
+ activeSpeakerFeed?.off(CallFeedEvent.VolumeChanged, onVolumeChanged);
+ setState((prevState) => ({
+ ...prevState,
+ activeSpeakerVolume: -Infinity,
+ }));
+ };
+ }, [activeSpeakerUserId, onVolumeChanged, userMediaFeeds, groupCall]);
+
const startTalking = useCallback(async () => {
if (pttButtonHeld) return;
@@ -211,6 +247,17 @@ export const usePTT = (
setMicMuteWrapper(true);
}, [setMicMuteWrapper]);
+ // separate state for connected: we set it separately from other things
+ // in the client sync callback
+ const [connected, setConnected] = useState(true);
+
+ const onClientSync = useCallback(
+ (syncState: SyncState) => {
+ setConnected(syncState !== SyncState.Error);
+ },
+ [setConnected]
+ );
+
useEffect(() => {
function onKeyDown(event: KeyboardEvent): void {
if (event.code === "Space") {
@@ -260,8 +307,18 @@ export const usePTT = (
pttButtonHeld,
enablePTTButton,
setMicMuteWrapper,
+ client,
+ onClientSync,
]);
+ useEffect(() => {
+ client.on(ClientEvent.Sync, onClientSync);
+
+ return () => {
+ client.removeListener(ClientEvent.Sync, onClientSync);
+ };
+ }, [client, onClientSync]);
+
const setTalkOverEnabled = useCallback((talkOverEnabled) => {
setState((prevState) => ({
...prevState,
@@ -275,8 +332,10 @@ export const usePTT = (
talkOverEnabled,
setTalkOverEnabled,
activeSpeakerUserId,
+ activeSpeakerVolume,
startTalking,
stopTalking,
transmitBlocked,
+ connected,
};
};
diff --git a/src/settings/SettingsModal.jsx b/src/settings/SettingsModal.jsx
index 3868864..60665c6 100644
--- a/src/settings/SettingsModal.jsx
+++ b/src/settings/SettingsModal.jsx
@@ -24,12 +24,13 @@ import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
import { SelectInput } from "../input/SelectInput";
import { Item } from "@react-stately/collections";
import { useMediaHandler } from "./useMediaHandler";
+import { useSpatialAudio, useShowInspector } from "./useSetting";
import { FieldRow, InputField } from "../input/Input";
import { Button } from "../button";
import { useDownloadDebugLog } from "./submit-rageshake";
import { Body } from "../typography/Typography";
-export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
+export const SettingsModal = (props) => {
const {
audioInput,
audioInputs,
@@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
audioOutputs,
setAudioOutput,
} = useMediaHandler();
+ const [spatialAudio, setSpatialAudio] = useSpatialAudio();
+ const [showInspector, setShowInspector] = useShowInspector();
const downloadDebugLog = useDownloadDebugLog();
@@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
isDismissable
mobileFullScreen
className={styles.settingsModal}
- {...rest}
+ {...props}
>
)}
+
+ setSpatialAudio(e.target.checked)}
+ />
+
);
-}
+};
diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts
new file mode 100644
index 0000000..b0db79c
--- /dev/null
+++ b/src/settings/useSetting.ts
@@ -0,0 +1,56 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { useMemo, useState, useEffect, useCallback } from "react";
+
+// Bus to notify other useSetting consumers when a setting is changed
+const settingsBus = new EventEmitter();
+
+// Like useState, but reads from and persists the value to localStorage
+const useSetting = (
+ name: string,
+ defaultValue: T
+): [T, (value: T) => void] => {
+ const key = useMemo(() => `matrix-setting-${name}`, [name]);
+
+ const [value, setValue] = useState(() => {
+ const item = localStorage.getItem(key);
+ return item == null ? defaultValue : JSON.parse(item);
+ });
+
+ useEffect(() => {
+ settingsBus.on(name, setValue);
+ return () => {
+ settingsBus.off(name, setValue);
+ };
+ }, [name, setValue]);
+
+ return [
+ value,
+ useCallback(
+ (newValue: T) => {
+ setValue(newValue);
+ localStorage.setItem(key, JSON.stringify(newValue));
+ settingsBus.emit(name, newValue);
+ },
+ [name, key, setValue]
+ ),
+ ];
+};
+
+export const useSpatialAudio = () => useSetting("spatial-audio", false);
+export const useShowInspector = () => useSetting("show-inspector", false);
diff --git a/src/video-grid/VideoTile.jsx b/src/video-grid/VideoTile.jsx
index 90780f7..2dd4192 100644
--- a/src/video-grid/VideoTile.jsx
+++ b/src/video-grid/VideoTile.jsx
@@ -14,57 +14,63 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-import React from "react";
+import React, { forwardRef } from "react";
import { animated } from "@react-spring/web";
import classNames from "classnames";
import styles from "./VideoTile.module.css";
import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
-export function VideoTile({
- className,
- isLocal,
- speaking,
- audioMuted,
- noVideo,
- videoMuted,
- screenshare,
- avatar,
- name,
- showName,
- mediaRef,
- ...rest
-}) {
- return (
-
- {(videoMuted || noVideo) && (
- <>
-
- {avatar}
- >
- )}
- {screenshare ? (
-
- {`${name} is presenting`}
-
- ) : (
- (showName || audioMuted || (videoMuted && !noVideo)) && (
-
- {audioMuted && !(videoMuted && !noVideo) &&
}
- {videoMuted && !noVideo &&
}
- {showName &&
{name}}
+export const VideoTile = forwardRef(
+ (
+ {
+ className,
+ isLocal,
+ speaking,
+ audioMuted,
+ noVideo,
+ videoMuted,
+ screenshare,
+ avatar,
+ name,
+ showName,
+ mediaRef,
+ ...rest
+ },
+ ref
+ ) => {
+ return (
+
+ {(videoMuted || noVideo) && (
+ <>
+
+ {avatar}
+ >
+ )}
+ {screenshare ? (
+
+ {`${name} is presenting`}
- )
- )}
-
-
- );
-}
+ ) : (
+ (showName || audioMuted || (videoMuted && !noVideo)) && (
+
+ {audioMuted && !(videoMuted && !noVideo) && }
+ {videoMuted && !noVideo && }
+ {showName && {name}}
+
+ )
+ )}
+
+
+ );
+ }
+);
diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css
index f5e2d11..0948488 100644
--- a/src/video-grid/VideoTile.module.css
+++ b/src/video-grid/VideoTile.module.css
@@ -5,6 +5,10 @@
overflow: hidden;
cursor: pointer;
touch-action: none;
+
+ /* HACK: This has no visual effect due to the short duration, but allows the
+ JS to detect movement via the transform property for audio spatialization */
+ transition: transform 0.000000001s;
}
.videoTile * {
diff --git a/src/video-grid/VideoTileContainer.jsx b/src/video-grid/VideoTileContainer.jsx
index dddba38..a4cf400 100644
--- a/src/video-grid/VideoTileContainer.jsx
+++ b/src/video-grid/VideoTileContainer.jsx
@@ -17,7 +17,7 @@ limitations under the License.
import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
import React from "react";
import { useCallFeed } from "./useCallFeed";
-import { useMediaStream } from "./useMediaStream";
+import { useSpatialMediaStream } from "./useMediaStream";
import { useRoomMemberName } from "./useRoomMemberName";
import { VideoTile } from "./VideoTile";
@@ -28,6 +28,7 @@ export function VideoTileContainer({
getAvatar,
showName,
audioOutputDevice,
+ audioContext,
disableSpeakingIndicator,
...rest
}) {
@@ -42,7 +43,12 @@ export function VideoTileContainer({
member,
} = useCallFeed(item.callFeed);
const { rawDisplayName } = useRoomMemberName(member);
- const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
+ const [tileRef, mediaRef] = useSpatialMediaStream(
+ stream,
+ audioOutputDevice,
+ audioContext,
+ isLocal
+ );
// Firefox doesn't respect the disablePictureInPicture attribute
// https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@@ -57,6 +63,7 @@ export function VideoTileContainer({
screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
name={rawDisplayName}
showName={showName}
+ ref={tileRef}
mediaRef={mediaRef}
avatar={getAvatar && getAvatar(member, width, height)}
{...rest}
diff --git a/src/video-grid/useMediaStream.js b/src/video-grid/useMediaStream.js
index 15efcff..774c6dd 100644
--- a/src/video-grid/useMediaStream.js
+++ b/src/video-grid/useMediaStream.js
@@ -16,6 +16,8 @@ limitations under the License.
import { useRef, useEffect } from "react";
+import { useSpatialAudio } from "../settings/useSetting";
+
export function useMediaStream(stream, audioOutputDevice, mute = false) {
const mediaRef = useRef();
@@ -55,7 +57,8 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
mediaRef.current !== undefined
) {
console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
- mediaRef.current.setSinkId(audioOutputDevice);
+ // Chrome for Android doesn't support this
+ mediaRef.current.setSinkId?.(audioOutputDevice);
}
}, [audioOutputDevice]);
@@ -73,3 +76,69 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
return mediaRef;
}
+
+export const useSpatialMediaStream = (
+ stream,
+ audioOutputDevice,
+ audioContext,
+ mute = false
+) => {
+ const tileRef = useRef();
+ const [spatialAudio] = useSpatialAudio();
+ // If spatial audio is enabled, we handle audio separately from the video element
+ const mediaRef = useMediaStream(
+ stream,
+ audioOutputDevice,
+ spatialAudio || mute
+ );
+
+ const pannerNodeRef = useRef();
+ if (!pannerNodeRef.current) {
+ pannerNodeRef.current = new PannerNode(audioContext, {
+ panningModel: "HRTF",
+ refDistance: 3,
+ });
+ }
+
+ const sourceRef = useRef();
+
+ useEffect(() => {
+ if (spatialAudio && tileRef.current && !mute) {
+ if (!sourceRef.current) {
+ sourceRef.current = audioContext.createMediaStreamSource(stream);
+ }
+
+ const tile = tileRef.current;
+ const source = sourceRef.current;
+ const pannerNode = pannerNodeRef.current;
+
+ const updatePosition = () => {
+ const bounds = tile.getBoundingClientRect();
+ const windowSize = Math.max(window.innerWidth, window.innerHeight);
+ // Position the source relative to its placement in the window
+ pannerNodeRef.current.positionX.value =
+ (bounds.x + bounds.width / 2) / windowSize - 0.5;
+ pannerNodeRef.current.positionY.value =
+ (bounds.y + bounds.height / 2) / windowSize - 0.5;
+ // Put the source in front of the listener
+ pannerNodeRef.current.positionZ.value = -2;
+ };
+
+ updatePosition();
+ source.connect(pannerNode);
+ pannerNode.connect(audioContext.destination);
+ // HACK: We abuse the CSS transitionrun event to detect when the tile
+ // moves, because useMeasure, IntersectionObserver, etc. all have no
+ // ability to track changes in the CSS transform property
+ tile.addEventListener("transitionrun", updatePosition);
+
+ return () => {
+ tile.removeEventListener("transitionrun", updatePosition);
+ source.disconnect();
+ pannerNode.disconnect();
+ };
+ }
+ }, [stream, spatialAudio, audioContext, mute]);
+
+ return [tileRef, mediaRef];
+};