From 1ff9073a1afc038fd0913d25dd3b68eb2323a4fb Mon Sep 17 00:00:00 2001
From: David Baker
Date: Mon, 30 May 2022 12:14:25 +0100
Subject: [PATCH 01/11] Sort call feeds consistently when choosing active
speaker
---
src/room/usePTT.ts | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index 5965c54..cc42140 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -30,6 +30,17 @@ function getActiveSpeakerFeed(
): CallFeed | null {
const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
+ // make sure the feeds are in a deterministic order so every client picks
+ // the same one as the active speaker
+ const collator = new Intl.Collator("en", {
+ sensitivity: "variant",
+ usage: "sort",
+ ignorePunctuation: false,
+ });
+ activeSpeakerFeeds.sort((a: CallFeed, b: CallFeed): number =>
+ collator.compare(a.userId, b.userId)
+ );
+
let activeSpeakerFeed = null;
let highestPowerLevel = null;
for (const feed of activeSpeakerFeeds) {
From 21c7bb979ee1ade3bfb18b1435ea1960dd45c282 Mon Sep 17 00:00:00 2001
From: David Baker
Date: Mon, 30 May 2022 15:30:57 +0100
Subject: [PATCH 02/11] Convert useGroupCall to TS
---
src/room/{useGroupCall.js => useGroupCall.ts} | 59 ++++++++++++++-----
1 file changed, 45 insertions(+), 14 deletions(-)
rename src/room/{useGroupCall.js => useGroupCall.ts} (82%)
diff --git a/src/room/useGroupCall.js b/src/room/useGroupCall.ts
similarity index 82%
rename from src/room/useGroupCall.js
rename to src/room/useGroupCall.ts
index 51900c5..25c2179 100644
--- a/src/room/useGroupCall.js
+++ b/src/room/useGroupCall.ts
@@ -18,10 +18,33 @@ import { useCallback, useEffect, useState } from "react";
import {
GroupCallEvent,
GroupCallState,
+ GroupCall,
} from "matrix-js-sdk/src/webrtc/groupCall";
+import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+import { RoomMember } from "matrix-js-sdk/src/models/room-member";
+
import { usePageUnload } from "./usePageUnload";
-export function useGroupCall(groupCall) {
+interface State {
+ state: GroupCallState;
+ calls: MatrixCall[];
+ localCallFeed: CallFeed;
+ activeSpeaker: string;
+ userMediaFeeds: CallFeed[];
+ error: Error;
+ microphoneMuted: boolean;
+ localVideoMuted: boolean;
+ screenshareFeeds: CallFeed[];
+ localScreenshareFeed: CallFeed;
+ localDesktopCapturerSourceId: string;
+ isScreensharing: boolean;
+ requestingScreenshare: boolean;
+ participants: RoomMember[];
+ hasLocalParticipant: boolean;
+}
+
+export function useGroupCall(groupCall: GroupCall) {
const [
{
state,
@@ -41,20 +64,25 @@ export function useGroupCall(groupCall) {
requestingScreenshare,
},
setState,
- ] = useState({
+ ] = useState({
state: GroupCallState.LocalCallFeedUninitialized,
calls: [],
+ localCallFeed: null,
+ activeSpeaker: null,
userMediaFeeds: [],
+ error: null,
microphoneMuted: false,
localVideoMuted: false,
- screenshareFeeds: [],
isScreensharing: false,
+ screenshareFeeds: [],
+ localScreenshareFeed: null,
+ localDesktopCapturerSourceId: null,
requestingScreenshare: false,
participants: [],
hasLocalParticipant: false,
});
- const updateState = (state) =>
+ const updateState = (state: Partial) =>
setState((prevState) => ({ ...prevState, ...state }));
useEffect(() => {
@@ -75,25 +103,28 @@ export function useGroupCall(groupCall) {
});
}
- function onUserMediaFeedsChanged(userMediaFeeds) {
+ function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void {
updateState({
userMediaFeeds: [...userMediaFeeds],
});
}
- function onScreenshareFeedsChanged(screenshareFeeds) {
+ function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void {
updateState({
screenshareFeeds: [...screenshareFeeds],
});
}
- function onActiveSpeakerChanged(activeSpeaker) {
+ function onActiveSpeakerChanged(activeSpeaker: string): void {
updateState({
activeSpeaker: activeSpeaker,
});
}
- function onLocalMuteStateChanged(microphoneMuted, localVideoMuted) {
+ function onLocalMuteStateChanged(
+ microphoneMuted: boolean,
+ localVideoMuted: boolean
+ ): void {
updateState({
microphoneMuted,
localVideoMuted,
@@ -101,10 +132,10 @@ export function useGroupCall(groupCall) {
}
function onLocalScreenshareStateChanged(
- isScreensharing,
- localScreenshareFeed,
- localDesktopCapturerSourceId
- ) {
+ isScreensharing: boolean,
+ localScreenshareFeed: CallFeed,
+ localDesktopCapturerSourceId: string
+ ): void {
updateState({
isScreensharing,
localScreenshareFeed,
@@ -112,13 +143,13 @@ export function useGroupCall(groupCall) {
});
}
- function onCallsChanged(calls) {
+ function onCallsChanged(calls: MatrixCall[]): void {
updateState({
calls: [...calls],
});
}
- function onParticipantsChanged(participants) {
+ function onParticipantsChanged(participants: RoomMember[]): void {
updateState({
participants: [...participants],
hasLocalParticipant: groupCall.hasLocalParticipant(),
From 1164e6f1e7282e394a2ed7409f795e73707cff2d Mon Sep 17 00:00:00 2001
From: David Baker
Date: Mon, 30 May 2022 15:53:44 +0100
Subject: [PATCH 03/11] Add return type too
---
src/room/useGroupCall.ts | 26 +++++++++++++++++++++++++-
1 file changed, 25 insertions(+), 1 deletion(-)
diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts
index 25c2179..6816bcf 100644
--- a/src/room/useGroupCall.ts
+++ b/src/room/useGroupCall.ts
@@ -26,6 +26,30 @@ import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { usePageUnload } from "./usePageUnload";
+export interface UseGroupCallType {
+ state: GroupCallState;
+ calls: MatrixCall[];
+ localCallFeed: CallFeed;
+ activeSpeaker: string;
+ userMediaFeeds: CallFeed[];
+ microphoneMuted: boolean;
+ localVideoMuted: boolean;
+ error: Error;
+ initLocalCallFeed: () => void;
+ enter: () => void;
+ leave: () => void;
+ toggleLocalVideoMuted: () => void;
+ toggleMicrophoneMuted: () => void;
+ toggleScreensharing: () => void;
+ requestingScreenshare: boolean;
+ isScreensharing: boolean;
+ screenshareFeeds: CallFeed[];
+ localScreenshareFeed: CallFeed;
+ localDesktopCapturerSourceId: string;
+ participants: RoomMember[];
+ hasLocalParticipant: boolean;
+}
+
interface State {
state: GroupCallState;
calls: MatrixCall[];
@@ -44,7 +68,7 @@ interface State {
hasLocalParticipant: boolean;
}
-export function useGroupCall(groupCall: GroupCall) {
+export function useGroupCall(groupCall: GroupCall): UseGroupCallType {
const [
{
state,
From e9b963080cd3ddcd8829a21983ef2b40db9995f0 Mon Sep 17 00:00:00 2001
From: David Baker
Date: Mon, 30 May 2022 16:28:16 +0100
Subject: [PATCH 04/11] Show when connection is lost on PTT mode
---
src/room/PTTCallView.tsx | 9 +++++++--
src/room/usePTT.ts | 26 +++++++++++++++++++++++++-
2 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx
index 4b395f8..12157d2 100644
--- a/src/room/PTTCallView.tsx
+++ b/src/room/PTTCallView.tsx
@@ -44,8 +44,11 @@ function getPromptText(
activeSpeakerIsLocalUser: boolean,
talkOverEnabled: boolean,
activeSpeakerUserId: string,
- activeSpeakerDisplayName: string
+ activeSpeakerDisplayName: string,
+ connected: boolean
): string {
+ if (!connected) return "Connection Lost";
+
const isTouchScreen = Boolean(window.ontouchstart !== undefined);
if (showTalkOverError) {
@@ -127,6 +130,7 @@ export const PTTCallView: React.FC = ({
startTalking,
stopTalking,
transmitBlocked,
+ connected,
} = usePTT(
client,
groupCall,
@@ -234,7 +238,8 @@ export const PTTCallView: React.FC = ({
activeSpeakerIsLocalUser,
talkOverEnabled,
activeSpeakerUserId,
- activeSpeakerDisplayName
+ activeSpeakerDisplayName,
+ connected
)}
{userMediaFeeds.map((callFeed) => (
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index 5965c54..328ea44 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -15,10 +15,11 @@ limitations under the License.
*/
import { useCallback, useEffect, useState } from "react";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
import { logger } from "matrix-js-sdk/src/logger";
+import { SyncState } from "matrix-js-sdk/src/sync";
import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
@@ -52,6 +53,11 @@ export interface PTTState {
startTalking: () => void;
stopTalking: () => void;
transmitBlocked: boolean;
+ // connected is actually an indication of whether we're connected to the HS
+ // (ie. the client's syncing state) rather than media connection, since
+ // it's peer to peer so we can't really say once peer is 'disconnected' if
+ // there's only one other person in the call and they've lost Internet.
+ connected: boolean;
}
export const usePTT = (
@@ -211,6 +217,17 @@ export const usePTT = (
setMicMuteWrapper(true);
}, [setMicMuteWrapper]);
+ // separate state for connected: we set it separately from other things
+ // in the client sync callback
+ const [connected, setConnected] = useState(true);
+
+ const onClientSync = useCallback(
+ (syncState: SyncState) => {
+ setConnected(syncState !== SyncState.Error);
+ },
+ [setConnected]
+ );
+
useEffect(() => {
function onKeyDown(event: KeyboardEvent): void {
if (event.code === "Space") {
@@ -245,10 +262,14 @@ export const usePTT = (
window.addEventListener("keyup", onKeyUp);
window.addEventListener("blur", onBlur);
+ client.on(ClientEvent.Sync, onClientSync);
+
return () => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("blur", onBlur);
+
+ client.removeListener(ClientEvent.Sync, onClientSync);
};
}, [
groupCall,
@@ -260,6 +281,8 @@ export const usePTT = (
pttButtonHeld,
enablePTTButton,
setMicMuteWrapper,
+ client,
+ onClientSync,
]);
const setTalkOverEnabled = useCallback((talkOverEnabled) => {
@@ -278,5 +301,6 @@ export const usePTT = (
startTalking,
stopTalking,
transmitBlocked,
+ connected,
};
};
From c6b90803f8e8552733f670bbd5e04d7cf7fa8bfc Mon Sep 17 00:00:00 2001
From: Robin Townsend
Date: Tue, 31 May 2022 10:43:05 -0400
Subject: [PATCH 05/11] Add spatial audio capabilities
---
src/room/GroupCallView.jsx | 19 -----
src/room/InCallView.jsx | 12 ++--
src/room/LobbyView.jsx | 4 --
src/room/OverflowMenu.jsx | 10 +--
src/room/PTTCallView.tsx | 6 --
src/room/VideoPreview.jsx | 4 --
src/settings/SettingsModal.jsx | 18 ++++-
src/settings/useSetting.ts | 56 +++++++++++++++
src/video-grid/VideoTile.jsx | 100 ++++++++++++++------------
src/video-grid/VideoTile.module.css | 4 ++
src/video-grid/VideoTileContainer.jsx | 11 ++-
src/video-grid/useMediaStream.js | 60 ++++++++++++++++
12 files changed, 205 insertions(+), 99 deletions(-)
create mode 100644 src/settings/useSetting.ts
diff --git a/src/room/GroupCallView.jsx b/src/room/GroupCallView.jsx
index dcd9da3..2809b82 100644
--- a/src/room/GroupCallView.jsx
+++ b/src/room/GroupCallView.jsx
@@ -33,19 +33,6 @@ export function GroupCallView({
roomId,
groupCall,
}) {
- const [showInspector, setShowInspector] = useState(
- () => !!localStorage.getItem("matrix-group-call-inspector")
- );
- const onChangeShowInspector = useCallback((show) => {
- setShowInspector(show);
-
- if (show) {
- localStorage.setItem("matrix-group-call-inspector", "true");
- } else {
- localStorage.removeItem("matrix-group-call-inspector");
- }
- }, []);
-
const {
state,
error,
@@ -104,8 +91,6 @@ export function GroupCallView({
participants={participants}
userMediaFeeds={userMediaFeeds}
onLeave={onLeave}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
/>
);
} else {
@@ -126,8 +111,6 @@ export function GroupCallView({
isScreensharing={isScreensharing}
localScreenshareFeed={localScreenshareFeed}
screenshareFeeds={screenshareFeeds}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
roomId={roomId}
/>
);
@@ -156,8 +139,6 @@ export function GroupCallView({
localVideoMuted={localVideoMuted}
toggleLocalVideoMuted={toggleLocalVideoMuted}
toggleMicrophoneMuted={toggleMicrophoneMuted}
- setShowInspector={onChangeShowInspector}
- showInspector={showInspector}
roomId={roomId}
/>
);
diff --git a/src/room/InCallView.jsx b/src/room/InCallView.jsx
index a474963..2c6240e 100644
--- a/src/room/InCallView.jsx
+++ b/src/room/InCallView.jsx
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-import React, { useCallback, useMemo } from "react";
+import React, { useCallback, useMemo, useRef } from "react";
import styles from "./InCallView.module.css";
import {
HangupButton,
@@ -34,6 +34,7 @@ import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { usePreventScroll } from "@react-aria/overlays";
import { useMediaHandler } from "../settings/useMediaHandler";
+import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
@@ -57,14 +58,16 @@ export function InCallView({
toggleScreensharing,
isScreensharing,
screenshareFeeds,
- setShowInspector,
- showInspector,
roomId,
}) {
usePreventScroll();
const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
const { audioOutput } = useMediaHandler();
+ const [showInspector] = useShowInspector();
+
+ const audioContext = useRef();
+ if (!audioContext.current) audioContext.current = new AudioContext();
const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
useModalTriggerState();
@@ -151,6 +154,7 @@ export function InCallView({
getAvatar={renderAvatar}
showName={items.length > 2 || item.focused}
audioOutputDevice={audioOutput}
+ audioContext={audioContext.current}
disableSpeakingIndicator={items.length < 3}
{...rest}
/>
@@ -169,8 +173,6 @@ export function InCallView({
diff --git a/src/room/OverflowMenu.jsx b/src/room/OverflowMenu.jsx
index 281995c..c5810f0 100644
--- a/src/room/OverflowMenu.jsx
+++ b/src/room/OverflowMenu.jsx
@@ -31,8 +31,6 @@ import { FeedbackModal } from "./FeedbackModal";
export function OverflowMenu({
roomId,
- setShowInspector,
- showInspector,
inCall,
groupCall,
showInvite,
@@ -88,13 +86,7 @@ export function OverflowMenu({
)}
- {settingsModalState.isOpen && (
-
- )}
+ {settingsModalState.isOpen && }
{inviteModalState.isOpen && (
)}
diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx
index 4b395f8..795c249 100644
--- a/src/room/PTTCallView.tsx
+++ b/src/room/PTTCallView.tsx
@@ -84,8 +84,6 @@ interface Props {
participants: RoomMember[];
userMediaFeeds: CallFeed[];
onLeave: () => void;
- setShowInspector: (boolean) => void;
- showInspector: boolean;
}
export const PTTCallView: React.FC = ({
@@ -97,8 +95,6 @@ export const PTTCallView: React.FC = ({
participants,
userMediaFeeds,
onLeave,
- setShowInspector,
- showInspector,
}) => {
const { modalState: inviteModalState, modalProps: inviteModalProps } =
useModalTriggerState();
@@ -189,8 +185,6 @@ export const PTTCallView: React.FC = ({
{
const {
audioInput,
audioInputs,
@@ -41,6 +42,8 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
audioOutputs,
setAudioOutput,
} = useMediaHandler();
+ const [spatialAudio, setSpatialAudio] = useSpatialAudio();
+ const [showInspector, setShowInspector] = useShowInspector();
const downloadDebugLog = useDownloadDebugLog();
@@ -50,7 +53,7 @@ export function SettingsModal({ setShowInspector, showInspector, ...rest }) {
isDismissable
mobileFullScreen
className={styles.settingsModal}
- {...rest}
+ {...props}
>
)}
+
+ setSpatialAudio(e.target.checked)}
+ />
+
);
-}
+};
diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts
new file mode 100644
index 0000000..b0db79c
--- /dev/null
+++ b/src/settings/useSetting.ts
@@ -0,0 +1,56 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { EventEmitter } from "events";
+import { useMemo, useState, useEffect, useCallback } from "react";
+
+// Bus to notify other useSetting consumers when a setting is changed
+const settingsBus = new EventEmitter();
+
+// Like useState, but reads from and persists the value to localStorage
+const useSetting = (
+ name: string,
+ defaultValue: T
+): [T, (value: T) => void] => {
+ const key = useMemo(() => `matrix-setting-${name}`, [name]);
+
+ const [value, setValue] = useState(() => {
+ const item = localStorage.getItem(key);
+ return item == null ? defaultValue : JSON.parse(item);
+ });
+
+ useEffect(() => {
+ settingsBus.on(name, setValue);
+ return () => {
+ settingsBus.off(name, setValue);
+ };
+ }, [name, setValue]);
+
+ return [
+ value,
+ useCallback(
+ (newValue: T) => {
+ setValue(newValue);
+ localStorage.setItem(key, JSON.stringify(newValue));
+ settingsBus.emit(name, newValue);
+ },
+ [name, key, setValue]
+ ),
+ ];
+};
+
+export const useSpatialAudio = () => useSetting("spatial-audio", false);
+export const useShowInspector = () => useSetting("show-inspector", false);
diff --git a/src/video-grid/VideoTile.jsx b/src/video-grid/VideoTile.jsx
index 90780f7..2dd4192 100644
--- a/src/video-grid/VideoTile.jsx
+++ b/src/video-grid/VideoTile.jsx
@@ -14,57 +14,63 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
-import React from "react";
+import React, { forwardRef } from "react";
import { animated } from "@react-spring/web";
import classNames from "classnames";
import styles from "./VideoTile.module.css";
import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
-export function VideoTile({
- className,
- isLocal,
- speaking,
- audioMuted,
- noVideo,
- videoMuted,
- screenshare,
- avatar,
- name,
- showName,
- mediaRef,
- ...rest
-}) {
- return (
-
- {(videoMuted || noVideo) && (
- <>
-
- {avatar}
- >
- )}
- {screenshare ? (
-
- {`${name} is presenting`}
-
- ) : (
- (showName || audioMuted || (videoMuted && !noVideo)) && (
-
- {audioMuted && !(videoMuted && !noVideo) &&
}
- {videoMuted && !noVideo &&
}
- {showName &&
{name}}
+export const VideoTile = forwardRef(
+ (
+ {
+ className,
+ isLocal,
+ speaking,
+ audioMuted,
+ noVideo,
+ videoMuted,
+ screenshare,
+ avatar,
+ name,
+ showName,
+ mediaRef,
+ ...rest
+ },
+ ref
+ ) => {
+ return (
+
+ {(videoMuted || noVideo) && (
+ <>
+
+ {avatar}
+ >
+ )}
+ {screenshare ? (
+
+ {`${name} is presenting`}
- )
- )}
-
-
- );
-}
+ ) : (
+ (showName || audioMuted || (videoMuted && !noVideo)) && (
+
+ {audioMuted && !(videoMuted && !noVideo) && }
+ {videoMuted && !noVideo && }
+ {showName && {name}}
+
+ )
+ )}
+
+
+ );
+ }
+);
diff --git a/src/video-grid/VideoTile.module.css b/src/video-grid/VideoTile.module.css
index f5e2d11..0948488 100644
--- a/src/video-grid/VideoTile.module.css
+++ b/src/video-grid/VideoTile.module.css
@@ -5,6 +5,10 @@
overflow: hidden;
cursor: pointer;
touch-action: none;
+
+ /* HACK: This has no visual effect due to the short duration, but allows the
+ JS to detect movement via the transform property for audio spatialization */
+ transition: transform 0.000000001s;
}
.videoTile * {
diff --git a/src/video-grid/VideoTileContainer.jsx b/src/video-grid/VideoTileContainer.jsx
index dddba38..a4cf400 100644
--- a/src/video-grid/VideoTileContainer.jsx
+++ b/src/video-grid/VideoTileContainer.jsx
@@ -17,7 +17,7 @@ limitations under the License.
import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
import React from "react";
import { useCallFeed } from "./useCallFeed";
-import { useMediaStream } from "./useMediaStream";
+import { useSpatialMediaStream } from "./useMediaStream";
import { useRoomMemberName } from "./useRoomMemberName";
import { VideoTile } from "./VideoTile";
@@ -28,6 +28,7 @@ export function VideoTileContainer({
getAvatar,
showName,
audioOutputDevice,
+ audioContext,
disableSpeakingIndicator,
...rest
}) {
@@ -42,7 +43,12 @@ export function VideoTileContainer({
member,
} = useCallFeed(item.callFeed);
const { rawDisplayName } = useRoomMemberName(member);
- const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal);
+ const [tileRef, mediaRef] = useSpatialMediaStream(
+ stream,
+ audioOutputDevice,
+ audioContext,
+ isLocal
+ );
// Firefox doesn't respect the disablePictureInPicture attribute
// https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
@@ -57,6 +63,7 @@ export function VideoTileContainer({
screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
name={rawDisplayName}
showName={showName}
+ ref={tileRef}
mediaRef={mediaRef}
avatar={getAvatar && getAvatar(member, width, height)}
{...rest}
diff --git a/src/video-grid/useMediaStream.js b/src/video-grid/useMediaStream.js
index 15efcff..0f8c3bf 100644
--- a/src/video-grid/useMediaStream.js
+++ b/src/video-grid/useMediaStream.js
@@ -16,6 +16,8 @@ limitations under the License.
import { useRef, useEffect } from "react";
+import { useSpatialAudio } from "../settings/useSetting";
+
export function useMediaStream(stream, audioOutputDevice, mute = false) {
const mediaRef = useRef();
@@ -73,3 +75,61 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
return mediaRef;
}
+
+export const useSpatialMediaStream = (
+ stream,
+ audioOutputDevice,
+ audioContext,
+ mute = false
+) => {
+ const tileRef = useRef();
+ const [spatialAudio] = useSpatialAudio();
+ // If spatial audio is enabled, we handle mute state separately from the video element
+ const mediaRef = useMediaStream(
+ stream,
+ audioOutputDevice,
+ spatialAudio || mute
+ );
+
+ const pannerNodeRef = useRef();
+ if (!pannerNodeRef.current) {
+ pannerNodeRef.current = new PannerNode(audioContext, {
+ panningModel: "HRTF",
+ });
+ }
+
+ useEffect(() => {
+ if (spatialAudio && tileRef.current && mediaRef.current && !mute) {
+ const tile = tileRef.current;
+ const pannerNode = pannerNodeRef.current;
+
+ const source = audioContext.createMediaElementSource(mediaRef.current);
+ const updatePosition = () => {
+ const bounds = tile.getBoundingClientRect();
+ const windowSize = Math.max(window.innerWidth, window.innerHeight);
+ // Position the source relative to its placement in the window
+ pannerNodeRef.current.positionX.value =
+ (bounds.x + bounds.width / 2) / windowSize - 0.5;
+ pannerNodeRef.current.positionY.value =
+ (bounds.y + bounds.height / 2) / windowSize - 0.5;
+ // Put the source in front of the listener
+ pannerNodeRef.current.positionZ.value = -2;
+ };
+
+ source.connect(pannerNode);
+ pannerNode.connect(audioContext.destination);
+ // HACK: We abuse the CSS transitionrun event to detect when the tile
+ // moves, because useMeasure, IntersectionObserver, etc. all have no
+ // ability to track changes in the CSS transform property
+ tile.addEventListener("transitionrun", updatePosition);
+
+ return () => {
+ tile.removeEventListener("transitionrun", updatePosition);
+ source.disconnect();
+ pannerNode.disconnect();
+ };
+ }
+ }, [spatialAudio, audioContext, mediaRef, mute]);
+
+ return [tileRef, mediaRef];
+};
From da3d038547337f006911a266a2af1f97f49a657f Mon Sep 17 00:00:00 2001
From: Robin Townsend
Date: Tue, 31 May 2022 16:11:39 -0400
Subject: [PATCH 06/11] Make it work on Chrome
---
src/video-grid/useMediaStream.js | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/src/video-grid/useMediaStream.js b/src/video-grid/useMediaStream.js
index 0f8c3bf..0ae058b 100644
--- a/src/video-grid/useMediaStream.js
+++ b/src/video-grid/useMediaStream.js
@@ -84,7 +84,7 @@ export const useSpatialMediaStream = (
) => {
const tileRef = useRef();
const [spatialAudio] = useSpatialAudio();
- // If spatial audio is enabled, we handle mute state separately from the video element
+ // If spatial audio is enabled, we handle audio separately from the video element
const mediaRef = useMediaStream(
stream,
audioOutputDevice,
@@ -95,15 +95,22 @@ export const useSpatialMediaStream = (
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
+ refDistance: 3,
});
}
+ const sourceRef = useRef();
+
useEffect(() => {
- if (spatialAudio && tileRef.current && mediaRef.current && !mute) {
+ if (spatialAudio && tileRef.current && !mute) {
+ if (!sourceRef.current) {
+ sourceRef.current = audioContext.createMediaStreamSource(stream);
+ }
+
const tile = tileRef.current;
+ const source = sourceRef.current;
const pannerNode = pannerNodeRef.current;
- const source = audioContext.createMediaElementSource(mediaRef.current);
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
@@ -116,6 +123,7 @@ export const useSpatialMediaStream = (
pannerNodeRef.current.positionZ.value = -2;
};
+ updatePosition();
source.connect(pannerNode);
pannerNode.connect(audioContext.destination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
@@ -129,7 +137,7 @@ export const useSpatialMediaStream = (
pannerNode.disconnect();
};
}
- }, [spatialAudio, audioContext, mediaRef, mute]);
+ }, [stream, spatialAudio, audioContext, mute]);
return [tileRef, mediaRef];
};
From e21094b5254192a9e86f29b27fb38dca563575c0 Mon Sep 17 00:00:00 2001
From: Robin Townsend
Date: Tue, 31 May 2022 16:21:35 -0400
Subject: [PATCH 07/11] Fix crash when setting audio output on Chrome for
Android
---
src/video-grid/useMediaStream.js | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/src/video-grid/useMediaStream.js b/src/video-grid/useMediaStream.js
index 15efcff..2432a73 100644
--- a/src/video-grid/useMediaStream.js
+++ b/src/video-grid/useMediaStream.js
@@ -55,7 +55,8 @@ export function useMediaStream(stream, audioOutputDevice, mute = false) {
mediaRef.current !== undefined
) {
console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
- mediaRef.current.setSinkId(audioOutputDevice);
+ // Chrome for Android doesn't support this
+ mediaRef.current.setSinkId?.(audioOutputDevice);
}
}, [audioOutputDevice]);
From 14fbddf78011296097b2111c3f3dd199967e9961 Mon Sep 17 00:00:00 2001
From: Robin Townsend
Date: Tue, 31 May 2022 18:08:42 -0400
Subject: [PATCH 08/11] Make PTTButton feel more clickable
---
src/room/PTTButton.module.css | 2 ++
1 file changed, 2 insertions(+)
diff --git a/src/room/PTTButton.module.css b/src/room/PTTButton.module.css
index 1a8dd17..d6b7a76 100644
--- a/src/room/PTTButton.module.css
+++ b/src/room/PTTButton.module.css
@@ -9,12 +9,14 @@
background-color: #21262c;
position: relative;
padding: 0;
+ cursor: pointer;
}
.talking {
background-color: #0dbd8b;
box-shadow: 0px 0px 0px 17px rgba(13, 189, 139, 0.2),
0px 0px 0px 34px rgba(13, 189, 139, 0.2);
+ cursor: unset;
}
.error {
From 771424cbf0d266d95543bb4bc84a058570d777ac Mon Sep 17 00:00:00 2001
From: David Baker
Date: Wed, 1 Jun 2022 10:11:02 +0100
Subject: [PATCH 09/11] Expand comment
---
src/room/usePTT.ts | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index cc42140..3446ff6 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -31,7 +31,11 @@ function getActiveSpeakerFeed(
const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
// make sure the feeds are in a deterministic order so every client picks
- // the same one as the active speaker
+ // the same one as the active speaker. The custom sort function sorts
+ // by user ID, so needs a collator of some kind to compare. We make a
+ // specific one to help ensure every client sorts the same way
+ // although of course user IDs shouldn't contain accented characters etc.
+ // anyway).
const collator = new Intl.Collator("en", {
sensitivity: "variant",
usage: "sort",
From 64e30c89e393e3a773a1511496b3af99c0caf6cb Mon Sep 17 00:00:00 2001
From: David Baker
Date: Wed, 1 Jun 2022 10:13:20 +0100
Subject: [PATCH 10/11] Comment typo
Co-authored-by: Robin
---
src/room/usePTT.ts | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index 328ea44..1a72faa 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -55,7 +55,7 @@ export interface PTTState {
transmitBlocked: boolean;
// connected is actually an indication of whether we're connected to the HS
// (ie. the client's syncing state) rather than media connection, since
- // it's peer to peer so we can't really say once peer is 'disconnected' if
+ // it's peer to peer so we can't really say which peer is 'disconnected' if
// there's only one other person in the call and they've lost Internet.
connected: boolean;
}
From d9bd48b9a68de6c866114b3ffb0ec6f8e2a1518e Mon Sep 17 00:00:00 2001
From: David Baker
Date: Wed, 1 Jun 2022 10:21:44 +0100
Subject: [PATCH 11/11] Split out client sync listeber into separate useEffect
---
src/room/usePTT.ts | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/src/room/usePTT.ts b/src/room/usePTT.ts
index 1a72faa..8a1f916 100644
--- a/src/room/usePTT.ts
+++ b/src/room/usePTT.ts
@@ -262,14 +262,10 @@ export const usePTT = (
window.addEventListener("keyup", onKeyUp);
window.addEventListener("blur", onBlur);
- client.on(ClientEvent.Sync, onClientSync);
-
return () => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("blur", onBlur);
-
- client.removeListener(ClientEvent.Sync, onClientSync);
};
}, [
groupCall,
@@ -285,6 +281,14 @@ export const usePTT = (
onClientSync,
]);
+ useEffect(() => {
+ client.on(ClientEvent.Sync, onClientSync);
+
+ return () => {
+ client.removeListener(ClientEvent.Sync, onClientSync);
+ };
+ }, [client, onClientSync]);
+
const setTalkOverEnabled = useCallback((talkOverEnabled) => {
setState((prevState) => ({
...prevState,