From f4f5c1ed3173bd9358b885b2dd6ecc53d477eb86 Mon Sep 17 00:00:00 2001 From: Daniel Abramov Date: Fri, 26 May 2023 20:41:32 +0200 Subject: [PATCH] Start using LiveKit SDK for media devices This version is not supposed to properly work, this is a work in progress. Main changes: * Completely removed the PTT logic (for simplicity, it could be introduced later). * Abstracted away the work with the media devices. * Defined confined interfaces of the affected components so that they only get the data that they need without importing Matris JS SDK or LiveKit SDK, so that we can exchange their "backend" at any time. * Started using JS/TS SDK from LiveKit as well as their React SDK to define the state of the local media devices and local streams. --- package.json | 3 +- src/media-utils.ts | 89 -------- src/room/AudioPreview.module.css | 43 ---- src/room/AudioPreview.tsx | 100 --------- src/room/GroupCallView.tsx | 155 +++++--------- src/room/InCallView.tsx | 11 +- src/room/LobbyView.tsx | 114 +++------- src/room/OverflowMenu.tsx | 46 ++-- src/room/PTTButton.module.css | 56 ----- src/room/PTTButton.tsx | 247 ---------------------- src/room/PTTCallView.module.css | 130 ------------ src/room/PTTCallView.tsx | 316 ---------------------------- src/room/PTTFeed.module.css | 19 -- src/room/PTTFeed.tsx | 34 --- src/room/RoomPage.tsx | 19 +- src/room/VideoPreview.tsx | 141 +++++++------ src/room/devices/mediaDevices.ts | 18 ++ src/room/devices/useMediaDevices.ts | 149 +++++++++++++ src/room/useLiveKit.ts | 145 +++++++++++++ src/settings/SettingsModal.tsx | 82 +++----- src/settings/useMediaHandler.tsx | 271 ------------------------ yarn.lock | 61 +++++- 22 files changed, 579 insertions(+), 1670 deletions(-) delete mode 100644 src/media-utils.ts delete mode 100644 src/room/AudioPreview.module.css delete mode 100644 src/room/AudioPreview.tsx delete mode 100644 src/room/PTTButton.module.css delete mode 100644 src/room/PTTButton.tsx delete mode 100644 src/room/PTTCallView.module.css delete mode 100644 src/room/PTTCallView.tsx delete mode 100644 src/room/PTTFeed.module.css delete mode 100644 src/room/PTTFeed.tsx create mode 100644 src/room/devices/mediaDevices.ts create mode 100644 src/room/devices/useMediaDevices.ts create mode 100644 src/room/useLiveKit.ts delete mode 100644 src/settings/useMediaHandler.tsx diff --git a/package.json b/package.json index b95f552..6f616f4 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ }, "dependencies": { "@juggle/resize-observer": "^3.3.1", + "@livekit/components-react": "^1.0.3", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz", "@react-aria/button": "^3.3.4", "@react-aria/dialog": "^3.1.4", @@ -45,7 +46,7 @@ "i18next": "^21.10.0", "i18next-browser-languagedetector": "^6.1.8", "i18next-http-backend": "^1.4.4", - "livekit-client": "^1.9.6", + "livekit-client": "^1.9.7", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#b1757b4f9dfe8a1fbb5b8d9ed697ff8b8516413e", "matrix-widget-api": "^1.0.0", "mermaid": "^9.4.0-rc.2", diff --git a/src/media-utils.ts b/src/media-utils.ts deleted file mode 100644 index 382360b..0000000 --- a/src/media-utils.ts +++ /dev/null @@ -1,89 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { logger } from "matrix-js-sdk/src/logger"; - -/** - * Finds a media device with label matching 'deviceName' - * @param deviceName The label of the device to look for - * @param devices The list of devices to search - * @returns A matching media device or undefined if no matching device was found - */ -export async function findDeviceByName( - deviceName: string, - kind: MediaDeviceKind, - devices: MediaDeviceInfo[] -): Promise { - const deviceInfo = devices.find( - (d) => d.kind === kind && d.label === deviceName - ); - return deviceInfo?.deviceId; -} - -/** - * Gets the available audio input/output and video input devices - * from the browser: a wrapper around mediaDevices.enumerateDevices() - * that requests a stream and holds it while calling enumerateDevices(). - * This is because some browsers (Firefox) only return device labels when - * the app has an active user media stream. In Chrome, this will get a - * stream from the default camera which can mean, for example, that the - * light for the FaceTime camera turns on briefly even if you selected - * another camera. Once the Permissions API - * (https://developer.mozilla.org/en-US/docs/Web/API/Permissions_API) - * is ready for primetime, this should allow us to avoid this. - * - * @return The available media devices - */ -export async function getDevices(): Promise { - // First get the devices without their labels, to learn what kinds of streams - // we can request - let devices: MediaDeviceInfo[]; - try { - devices = await navigator.mediaDevices.enumerateDevices(); - } catch (error) { - logger.warn("Unable to refresh WebRTC devices", error); - devices = []; - } - - let stream: MediaStream | null = null; - try { - if (devices.some((d) => d.kind === "audioinput")) { - // Holding just an audio stream will be enough to get us all device labels - stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - } else if (devices.some((d) => d.kind === "videoinput")) { - // We have to resort to a video stream - stream = await navigator.mediaDevices.getUserMedia({ video: true }); - } - } catch (e) { - logger.info("Couldn't get media stream for enumerateDevices: failing"); - throw e; - } - - if (stream !== null) { - try { - return await navigator.mediaDevices.enumerateDevices(); - } catch (error) { - logger.warn("Unable to refresh WebRTC devices", error); - } finally { - for (const track of stream.getTracks()) { - track.stop(); - } - } - } - - // If all else failed, continue without device labels - return devices; -} diff --git a/src/room/AudioPreview.module.css b/src/room/AudioPreview.module.css deleted file mode 100644 index cf5e9c9..0000000 --- a/src/room/AudioPreview.module.css +++ /dev/null @@ -1,43 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.preview { - margin: 20px 0; - padding: 24px 20px; - border-radius: 8px; - width: calc(100% - 40px); - max-width: 414px; -} - -.inputField { - width: 100%; -} - -.inputField:last-child { - margin-bottom: 0; -} - -.microphonePermissions { - margin: 20px; - text-align: center; -} - -@media (min-width: 800px) { - .preview { - margin-top: 40px; - background-color: #21262c; - } -} diff --git a/src/room/AudioPreview.tsx b/src/room/AudioPreview.tsx deleted file mode 100644 index 8c7459d..0000000 --- a/src/room/AudioPreview.tsx +++ /dev/null @@ -1,100 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import React from "react"; -import { GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall"; -import { Item } from "@react-stately/collections"; -import { useTranslation } from "react-i18next"; - -import styles from "./AudioPreview.module.css"; -import { SelectInput } from "../input/SelectInput"; -import { Body } from "../typography/Typography"; - -interface Props { - state: GroupCallState; - roomName: string; - audioInput: string; - audioInputs: MediaDeviceInfo[]; - setAudioInput: (deviceId: string) => void; - audioOutput: string; - audioOutputs: MediaDeviceInfo[]; - setAudioOutput: (deviceId: string) => void; -} - -export function AudioPreview({ - state, - roomName, - audioInput, - audioInputs, - setAudioInput, - audioOutput, - audioOutputs, - setAudioOutput, -}: Props) { - const { t } = useTranslation(); - - return ( - <> -

{t("{{roomName}} - Walkie-talkie call", { roomName })}

-
- {state === GroupCallState.LocalCallFeedUninitialized && ( - - {t("Microphone permissions needed to join the call.")} - - )} - {state === GroupCallState.InitializingLocalCallFeed && ( - - {t("Accept microphone permissions to join the call.")} - - )} - {state === GroupCallState.LocalCallFeedInitialized && ( - <> - - {audioInputs.map(({ deviceId, label }, index) => ( - - {!!label && label.trim().length > 0 - ? label - : t("Microphone {{n}}", { n: index + 1 })} - - ))} - - {audioOutputs.length > 0 && ( - - {audioOutputs.map(({ deviceId, label }, index) => ( - - {!!label && label.trim().length > 0 - ? label - : t("Speaker {{n}}", { n: index + 1 })} - - ))} - - )} - - )} -
- - ); -} diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8867ed4..b0ec4e8 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -18,23 +18,21 @@ import React, { useCallback, useEffect, useState } from "react"; import { useHistory } from "react-router-dom"; import { GroupCall, GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall"; import { MatrixClient } from "matrix-js-sdk/src/client"; -import { logger } from "matrix-js-sdk/src/logger"; import { useTranslation } from "react-i18next"; import type { IWidgetApiRequest } from "matrix-widget-api"; -import { widget, ElementWidgetActions, JoinCallData } from "../widget"; +import { widget, ElementWidgetActions } from "../widget"; import { useGroupCall } from "./useGroupCall"; import { ErrorView, FullScreenView } from "../FullScreenView"; import { LobbyView } from "./LobbyView"; +import { MatrixInfo } from "./VideoPreview"; import { InCallView } from "./InCallView"; -import { PTTCallView } from "./PTTCallView"; import { CallEndedView } from "./CallEndedView"; -import { useRoomAvatar } from "./useRoomAvatar"; import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { useLocationNavigation } from "../useLocationNavigation"; import { PosthogAnalytics } from "../PosthogAnalytics"; -import { useMediaHandler } from "../settings/useMediaHandler"; -import { findDeviceByName, getDevices } from "../media-utils"; +import { useProfile } from "../profile/useProfile"; +import { useLiveKit } from "./useLiveKit"; declare global { interface Window { @@ -68,8 +66,6 @@ export function GroupCallView({ userMediaFeeds, microphoneMuted, localVideoMuted, - localCallFeed, - initLocalCallFeed, enter, leave, toggleLocalVideoMuted, @@ -84,8 +80,6 @@ export function GroupCallView({ } = useGroupCall(groupCall); const { t } = useTranslation(); - const { setAudioInput, setVideoInput } = useMediaHandler(); - const avatarUrl = useRoomAvatar(groupCall.room); useEffect(() => { window.groupCall = groupCall; @@ -94,54 +88,22 @@ export function GroupCallView({ }; }, [groupCall]); + const { displayName, avatarUrl } = useProfile(client); + + const matrixInfo: MatrixInfo = { + userName: displayName, + avatarUrl, + roomName: groupCall.room.name, + roomId: roomIdOrAlias, + }; + + // TODO: Pass the correct URL and the correct JWT token here. + const lkState = useLiveKit("", ""); + useEffect(() => { if (widget && preload) { // In preload mode, wait for a join action before entering const onJoin = async (ev: CustomEvent) => { - // Get the available devices so we can match the selected device - // to its ID. This involves getting a media stream (see docs on - // the function) so we only do it once and re-use the result. - const devices = await getDevices(); - - const { audioInput, videoInput } = ev.detail - .data as unknown as JoinCallData; - - if (audioInput !== null) { - const deviceId = await findDeviceByName( - audioInput, - "audioinput", - devices - ); - if (!deviceId) { - logger.warn("Unknown audio input: " + audioInput); - } else { - logger.debug( - `Found audio input ID ${deviceId} for name ${audioInput}` - ); - setAudioInput(deviceId); - } - } - - if (videoInput !== null) { - const deviceId = await findDeviceByName( - videoInput, - "videoinput", - devices - ); - if (!deviceId) { - logger.warn("Unknown video input: " + videoInput); - } else { - logger.debug( - `Found video input ID ${deviceId} for name ${videoInput}` - ); - setVideoInput(deviceId); - } - } - await Promise.all([ - groupCall.setMicrophoneMuted(audioInput === null), - groupCall.setLocalVideoMuted(videoInput === null), - ]); - await groupCall.enter(); PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); @@ -158,7 +120,7 @@ export function GroupCallView({ widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin); }; } - }, [groupCall, preload, setAudioInput, setVideoInput]); + }, [groupCall, preload]); useEffect(() => { if (isEmbedded && !preload) { @@ -225,46 +187,30 @@ export function GroupCallView({ if (error) { return ; } else if (state === GroupCallState.Entered) { - if (groupCall.isPtt) { - return ( - - ); - } else { - return ( - - ); - } + return ( + + ); } else if (left) { if (isPasswordlessUser) { return ; @@ -282,25 +228,18 @@ export function GroupCallView({

{t("Loading room…")}

); - } else { + } else if (lkState) { return ( ); + } else { + return null; } } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b072f64..074290f 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -50,7 +50,6 @@ import { Avatar } from "../Avatar"; import { UserMenuContainer } from "../UserMenuContainer"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; -import { useMediaHandler } from "../settings/useMediaHandler"; import { useShowInspector, useSpatialAudio } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { useAudioContext } from "../video-grid/useMediaStream"; @@ -64,6 +63,7 @@ import { ParticipantInfo } from "./useGroupCall"; import { TileDescriptor } from "../video-grid/TileDescriptor"; import { AudioSink } from "../video-grid/AudioSink"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; +import { MediaDevicesState } from "./devices/useMediaDevices"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -77,6 +77,7 @@ interface Props { participants: Map>; roomName: string; avatarUrl: string; + mediaDevices: MediaDevicesState; microphoneMuted: boolean; localVideoMuted: boolean; toggleLocalVideoMuted: () => void; @@ -99,6 +100,7 @@ export function InCallView({ participants, roomName, avatarUrl, + mediaDevices, microphoneMuted, localVideoMuted, toggleLocalVideoMuted, @@ -349,7 +351,6 @@ export function InCallView({ // audio rendering for feeds that we're displaying, which will need to be fixed // once we start having more participants than we can fit on a screen, but this // is a workaround for now. - const { audioOutput } = useMediaHandler(); const audioElements: JSX.Element[] = []; if (!spatialAudio || maximisedParticipant) { for (const item of items) { @@ -357,7 +358,7 @@ export function InCallView({ audioElements.push( ); @@ -389,9 +390,9 @@ export function InCallView({ )} {!maximisedParticipant && ( void; + matrixInfo: MatrixInfo; + mediaDevices: MediaDevicesState; + localMedia: LocalMediaInfo; + onEnter: (e: PressEvent) => void; - localCallFeed: CallFeed; - microphoneMuted: boolean; - toggleLocalVideoMuted: () => void; - toggleMicrophoneMuted: () => void; - localVideoMuted: boolean; - roomIdOrAlias: string; isEmbedded: boolean; hideHeader: boolean; } -export function LobbyView({ - client, - groupCall, - roomName, - avatarUrl, - state, - onInitLocalCallFeed, - onEnter, - localCallFeed, - microphoneMuted, - localVideoMuted, - toggleLocalVideoMuted, - toggleMicrophoneMuted, - roomIdOrAlias, - isEmbedded, - hideHeader, -}: Props) { + +export function LobbyView(props: Props) { const { t } = useTranslation(); - const { stream } = useCallFeed(localCallFeed); - const { - audioInput, - audioInputs, - setAudioInput, - audioOutput, - audioOutputs, - setAudioOutput, - } = useMediaHandler(); + useLocationNavigation(); - useEffect(() => { - onInitLocalCallFeed(); - }, [onInitLocalCallFeed]); - - useLocationNavigation(state === GroupCallState.InitializingLocalCallFeed); - - const joinCallButtonRef = useRef(); - - useEffect(() => { - if (state === GroupCallState.LocalCallFeedInitialized) { + const joinCallButtonRef = React.useRef(); + React.useEffect(() => { + if (joinCallButtonRef.current) { joinCallButtonRef.current.focus(); } - }, [state]); + }, [joinCallButtonRef]); return (
- {!hideHeader && ( + {!props.hideHeader && (
- + @@ -106,44 +66,24 @@ export function LobbyView({ )}
- {groupCall.isPtt ? ( - - ) : ( - - )} + Or @@ -151,7 +91,7 @@ export function LobbyView({
- {!isEmbedded && ( + {!props.isEmbedded && ( {t("Take me Home")} diff --git a/src/room/OverflowMenu.tsx b/src/room/OverflowMenu.tsx index 0920f2d..0e0c3d9 100644 --- a/src/room/OverflowMenu.tsx +++ b/src/room/OverflowMenu.tsx @@ -16,7 +16,6 @@ limitations under the License. import React, { useCallback } from "react"; import { Item } from "@react-stately/collections"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { OverlayTriggerState } from "@react-stately/overlays"; import { useTranslation } from "react-i18next"; @@ -33,11 +32,13 @@ import { InviteModal } from "./InviteModal"; import { TooltipTrigger } from "../Tooltip"; import { FeedbackModal } from "./FeedbackModal"; import { Config } from "../config/Config"; +import { MediaDevicesState } from "./devices/useMediaDevices"; interface Props { - roomIdOrAlias: string; + roomId: string; + mediaDevices: MediaDevicesState; + inCall: boolean; - groupCall: GroupCall; showInvite: boolean; feedbackModalState: OverlayTriggerState; feedbackModalProps: { @@ -46,16 +47,8 @@ interface Props { }; } -export function OverflowMenu({ - roomIdOrAlias, - inCall, - groupCall, - showInvite, - feedbackModalState, - feedbackModalProps, -}: Props) { +export function OverflowMenu(props: Props) { const { t } = useTranslation(); - const { modalState: inviteModalState, modalProps: inviteModalProps, @@ -89,11 +82,11 @@ export function OverflowMenu({ settingsModalState.open(); break; case "feedback": - feedbackModalState.open(); + props.feedbackModalState.open(); break; } }, - [feedbackModalState, inviteModalState, settingsModalState] + [props.feedbackModalState, inviteModalState, settingsModalState] ); const tooltip = useCallback(() => t("More"), [t]); @@ -106,9 +99,9 @@ export function OverflowMenu({ - {(props: JSX.IntrinsicAttributes) => ( - - {showInvite && ( + {(attr: JSX.IntrinsicAttributes) => ( + + {props.showInvite && ( {t("Invite people")} @@ -127,15 +120,20 @@ export function OverflowMenu({ )} - {settingsModalState.isOpen && } - {inviteModalState.isOpen && ( - + {settingsModalState.isOpen && ( + )} - {feedbackModalState.isOpen && ( + {inviteModalState.isOpen && ( + + )} + {props.feedbackModalState.isOpen && ( )} diff --git a/src/room/PTTButton.module.css b/src/room/PTTButton.module.css deleted file mode 100644 index cf21af2..0000000 --- a/src/room/PTTButton.module.css +++ /dev/null @@ -1,56 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.pttButton { - width: 100vw; - aspect-ratio: 1; - max-height: min(232px, calc(100vh - 16px)); - max-width: min(232px, calc(100vw - 16px)); - border-radius: 116px; - color: var(--primary-content); - border: 6px solid var(--accent); - background-color: #21262c; - position: relative; - padding: 0; - margin: 4px; - cursor: pointer; -} - -.micIcon { - max-height: 50%; -} - -.avatar { - /* Remove explicit size to allow avatar to scale with the button */ - width: 100% !important; - height: 100% !important; -} - -.talking { - background-color: var(--accent); - cursor: unset; -} - -.networkWaiting { - background-color: var(--tertiary-content); - border-color: var(--tertiary-content); - cursor: unset; -} - -.error { - background-color: var(--alert); - border-color: var(--alert); -} diff --git a/src/room/PTTButton.tsx b/src/room/PTTButton.tsx deleted file mode 100644 index 1ff7b27..0000000 --- a/src/room/PTTButton.tsx +++ /dev/null @@ -1,247 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import React, { useCallback, useState, useRef } from "react"; -import classNames from "classnames"; -import { useSpring, animated } from "@react-spring/web"; -import { logger } from "@sentry/utils"; - -import styles from "./PTTButton.module.css"; -import { ReactComponent as MicIcon } from "../icons/Mic.svg"; -import { useEventTarget } from "../useEvents"; -import { Avatar } from "../Avatar"; -import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; -import { getSetting } from "../settings/useSetting"; - -interface Props { - enabled: boolean; - showTalkOverError: boolean; - activeSpeakerUserId: string; - activeSpeakerDisplayName: string; - activeSpeakerAvatarUrl: string; - activeSpeakerIsLocalUser: boolean; - activeSpeakerVolume: number; - size: number; - startTalking: () => void; - stopTalking: () => void; - networkWaiting: boolean; - enqueueNetworkWaiting: (value: boolean, delay: number) => void; - setNetworkWaiting: (value: boolean) => void; -} - -export const PTTButton: React.FC = ({ - enabled, - showTalkOverError, - activeSpeakerUserId, - activeSpeakerDisplayName, - activeSpeakerAvatarUrl, - activeSpeakerIsLocalUser, - activeSpeakerVolume, - size, - startTalking, - stopTalking, - networkWaiting, - enqueueNetworkWaiting, - setNetworkWaiting, -}) => { - const buttonRef = useRef(); - - const [activeTouchId, setActiveTouchId] = useState(null); - const [buttonHeld, setButtonHeld] = useState(false); - - const hold = useCallback(() => { - // This update is delayed so the user only sees it if latency is significant - if (buttonHeld) return; - setButtonHeld(true); - enqueueNetworkWaiting(true, 100); - startTalking(); - }, [enqueueNetworkWaiting, startTalking, buttonHeld]); - - const unhold = useCallback(() => { - if (!buttonHeld) return; - setButtonHeld(false); - setNetworkWaiting(false); - stopTalking(); - }, [setNetworkWaiting, stopTalking, buttonHeld]); - - const onMouseUp = useCallback(() => { - logger.info("Mouse up event: unholding PTT button"); - unhold(); - }, [unhold]); - - const onBlur = useCallback(() => { - logger.info("Blur event: unholding PTT button"); - unhold(); - }, [unhold]); - - const onButtonMouseDown = useCallback( - (e: React.MouseEvent) => { - e.preventDefault(); - hold(); - }, - [hold] - ); - - // These listeners go on the window so even if the user's cursor / finger - // leaves the button while holding it, the button stays pushed until - // they stop clicking / tapping. - useEventTarget(window, "mouseup", onMouseUp); - useEventTarget( - window, - "touchend", - useCallback( - (e: TouchEvent) => { - // ignore any ended touches that weren't the one pressing the - // button (bafflingly the TouchList isn't an iterable so we - // have to do this a really old-school way). - let touchFound = false; - for (let i = 0; i < e.changedTouches.length; ++i) { - if (e.changedTouches.item(i).identifier === activeTouchId) { - touchFound = true; - break; - } - } - if (!touchFound) return; - - logger.info("Touch event ended: unholding PTT button"); - - e.preventDefault(); - unhold(); - setActiveTouchId(null); - }, - [unhold, activeTouchId, setActiveTouchId] - ) - ); - - // This is a native DOM listener too because we want to preventDefault in it - // to stop also getting a click event, so we need it to be non-passive. - useEventTarget( - buttonRef.current, - "touchstart", - useCallback( - (e: TouchEvent) => { - e.preventDefault(); - - hold(); - setActiveTouchId(e.changedTouches.item(0).identifier); - }, - [hold, setActiveTouchId] - ), - { passive: false } - ); - - useEventTarget( - window, - "keydown", - useCallback( - (e: KeyboardEvent) => { - if (e.code === "Space") { - if (!enabled) return; - // Check if keyboard shortcuts are enabled - const keyboardShortcuts = getSetting("keyboard-shortcuts", true); - if (!keyboardShortcuts) { - return; - } - - e.preventDefault(); - - hold(); - } - }, - [enabled, hold] - ) - ); - useEventTarget( - window, - "keyup", - useCallback( - (e: KeyboardEvent) => { - if (e.code === "Space") { - // Check if keyboard shortcuts are enabled - const keyboardShortcuts = getSetting("keyboard-shortcuts", true); - if (!keyboardShortcuts) { - return; - } - - e.preventDefault(); - - logger.info("Keyup event for spacebar: unholding PTT button"); - - unhold(); - } - }, - [unhold] - ) - ); - - // TODO: We will need to disable this for a global PTT hotkey to work - useEventTarget(window, "blur", onBlur); - - const prefersReducedMotion = usePrefersReducedMotion(); - const { shadow } = useSpring({ - immediate: prefersReducedMotion, - shadow: prefersReducedMotion - ? activeSpeakerUserId - ? 17 - : 0 - : (Math.max(activeSpeakerVolume, -70) + 70) * 0.6, - config: { - clamp: true, - tension: 300, - }, - }); - const shadowColor = showTalkOverError - ? "var(--alert-20)" - : networkWaiting - ? "var(--tertiary-content-20)" - : "var(--accent-20)"; - - return ( - - `0px 0px 0px ${s}px ${shadowColor}, 0px 0px 0px ${ - 2 * s - }px ${shadowColor}` - ), - }} - onMouseDown={onButtonMouseDown} - ref={buttonRef} - > - {activeSpeakerIsLocalUser || !activeSpeakerUserId ? ( - - ) : ( - - )} - - ); -}; diff --git a/src/room/PTTCallView.module.css b/src/room/PTTCallView.module.css deleted file mode 100644 index 2a35db5..0000000 --- a/src/room/PTTCallView.module.css +++ /dev/null @@ -1,130 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.pttCallView { - position: relative; - display: flex; - flex-direction: column; - justify-content: center; - overflow: hidden; - min-height: 100%; - position: fixed; - height: 100%; - width: 100%; -} - -@media (hover: none) { - .pttCallView { - user-select: none; - } -} - -.center { - width: 100%; - display: flex; - flex: 1; - flex-direction: column; - align-items: center; -} - -.participants { - display: flex; - flex-direction: column; - margin: 20px; - text-align: center; -} - -.participants > p { - color: var(--secondary-content); - margin-bottom: 8px; -} - -.facepile { - align-self: center; -} - -.talkingInfo { - display: flex; - flex-shrink: 0; - flex-direction: column; - align-items: center; - margin-bottom: 20px; - height: 88px; -} - -.speakerIcon { - margin-right: 8px; -} - -.pttButtonContainer { - display: flex; - flex-direction: column; - align-items: center; - flex: 1; - justify-content: center; -} - -.actionTip { - margin-top: 20px; - margin-bottom: 20px; - font-size: var(--font-size-subtitle); -} - -.footer { - position: relative; - display: flex; - justify-content: center; - height: 64px; - margin-bottom: 20px; -} - -.footer > * { - margin-right: 30px; -} - -.footer > :last-child { - margin-right: 0px; -} - -@media (min-width: 800px) { - .participants { - margin-bottom: 67px; - } - - .talkingInfo { - margin-bottom: 38px; - } - - .center { - margin-top: 48px; - } - - .actionTip { - margin-top: 42px; - margin-bottom: 45px; - } - - .pttButtonContainer { - flex: 0; - margin-bottom: 0; - justify-content: flex-start; - } - - .footer { - flex: auto; - order: 4; - } -} diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx deleted file mode 100644 index a25f337..0000000 --- a/src/room/PTTCallView.tsx +++ /dev/null @@ -1,316 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import React, { useEffect, useMemo } from "react"; -import useMeasure from "react-use-measure"; -import { ResizeObserver } from "@juggle/resize-observer"; -import i18n from "i18next"; -import { MatrixClient } from "matrix-js-sdk/src/client"; -import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; -import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; -import { useTranslation } from "react-i18next"; - -import { useDelayedState } from "../useDelayedState"; -import { useModalTriggerState } from "../Modal"; -import { InviteModal } from "./InviteModal"; -import { HangupButton, InviteButton } from "../button"; -import { Header, LeftNav, RightNav, RoomSetupHeaderInfo } from "../Header"; -import styles from "./PTTCallView.module.css"; -import { Facepile } from "../Facepile"; -import { PTTButton } from "./PTTButton"; -import { PTTFeed } from "./PTTFeed"; -import { useMediaHandler } from "../settings/useMediaHandler"; -import { usePTT } from "./usePTT"; -import { Timer } from "./Timer"; -import { Toggle } from "../input/Toggle"; -import { ReactComponent as AudioIcon } from "../icons/Audio.svg"; -import { usePTTSounds } from "../sound/usePttSounds"; -import { PTTClips } from "../sound/PTTClips"; -import { GroupCallInspector } from "./GroupCallInspector"; -import { OverflowMenu } from "./OverflowMenu"; -import { Size } from "../Avatar"; -import { ParticipantInfo } from "./useGroupCall"; - -function getPromptText( - networkWaiting: boolean, - showTalkOverError: boolean, - pttButtonHeld: boolean, - activeSpeakerIsLocalUser: boolean, - talkOverEnabled: boolean, - activeSpeakerUserId: string, - activeSpeakerDisplayName: string, - connected: boolean, - t: typeof i18n.t -): string { - if (!connected) return t("Connection lost"); - - const isTouchScreen = Boolean(window.ontouchstart !== undefined); - - if (networkWaiting) { - return t("Waiting for network"); - } - - if (showTalkOverError) { - return t("You can't talk at the same time"); - } - - if (pttButtonHeld && activeSpeakerIsLocalUser) { - if (isTouchScreen) { - return t("Release to stop"); - } else { - return t("Release spacebar key to stop"); - } - } - - if (talkOverEnabled && activeSpeakerUserId && !activeSpeakerIsLocalUser) { - if (isTouchScreen) { - return t("Press and hold to talk over {{name}}", { - name: activeSpeakerDisplayName, - }); - } else { - return t("Press and hold spacebar to talk over {{name}}", { - name: activeSpeakerDisplayName, - }); - } - } - - if (isTouchScreen) { - return t("Press and hold to talk"); - } else { - return t("Press and hold spacebar to talk"); - } -} - -interface Props { - client: MatrixClient; - roomIdOrAlias: string; - roomName: string; - avatarUrl: string; - groupCall: GroupCall; - participants: Map>; - userMediaFeeds: CallFeed[]; - onLeave: () => void; - isEmbedded: boolean; - hideHeader: boolean; -} - -export const PTTCallView: React.FC = ({ - client, - roomIdOrAlias, - roomName, - avatarUrl, - groupCall, - participants, - userMediaFeeds, - onLeave, - isEmbedded, - hideHeader, -}) => { - const { t } = useTranslation(); - const { modalState: inviteModalState, modalProps: inviteModalProps } = - useModalTriggerState(); - const { modalState: feedbackModalState, modalProps: feedbackModalProps } = - useModalTriggerState(); - const [containerRef, bounds] = useMeasure({ polyfill: ResizeObserver }); - const facepileSize = bounds.width < 800 ? Size.SM : Size.MD; - const showControls = bounds.height > 500; - const pttButtonSize = 232; - - const { audioOutput } = useMediaHandler(); - - const { - startTalkingLocalRef, - startTalkingRemoteRef, - blockedRef, - endTalkingRef, - playClip, - } = usePTTSounds(); - - const { - pttButtonHeld, - isAdmin, - talkOverEnabled, - setTalkOverEnabled, - activeSpeakerUserId, - activeSpeakerVolume, - startTalking, - stopTalking, - transmitBlocked, - connected, - } = usePTT(client, groupCall, userMediaFeeds, playClip); - - const participatingMembers = useMemo(() => { - const members: RoomMember[] = []; - for (const [member, deviceMap] of participants) { - // Repeat the member for as many devices as they're using - for (let i = 0; i < deviceMap.size; i++) members.push(member); - } - return members; - }, [participants]); - - const [talkingExpected, enqueueTalkingExpected, setTalkingExpected] = - useDelayedState(false); - const showTalkOverError = pttButtonHeld && transmitBlocked; - const networkWaiting = - talkingExpected && !activeSpeakerUserId && !showTalkOverError; - - const activeSpeakerIsLocalUser = activeSpeakerUserId === client.getUserId(); - const activeSpeakerUser = activeSpeakerUserId - ? client.getUser(activeSpeakerUserId) - : null; - const activeSpeakerAvatarUrl = activeSpeakerUser?.avatarUrl; - const activeSpeakerDisplayName = activeSpeakerUser - ? activeSpeakerUser.displayName - : ""; - - useEffect(() => { - setTalkingExpected(activeSpeakerIsLocalUser); - }, [activeSpeakerIsLocalUser, setTalkingExpected]); - - return ( -
- - - {!hideHeader && showControls && ( -
- - - - -
- )} -
- {/* Always render this because the window will become shorter when the on-screen - keyboard appears, so if we don't render it, the dialog will unmount. */} -
-
-

- {t("{{count}} people connected", { - count: participatingMembers.length, - })} -

- -
-
- - {!isEmbedded && } - inviteModalState.open()} /> -
-
- -
- {showControls && - (activeSpeakerUserId ? ( -
-

- {!activeSpeakerIsLocalUser && ( - - )} - {activeSpeakerIsLocalUser - ? t("Talking…") - : t("{{name}} is talking…", { - name: activeSpeakerDisplayName, - })} -

- -
- ) : ( -
- ))} - - {showControls && ( -

- {getPromptText( - networkWaiting, - showTalkOverError, - pttButtonHeld, - activeSpeakerIsLocalUser, - talkOverEnabled, - activeSpeakerUserId, - activeSpeakerDisplayName, - connected, - t - )} -

- )} - {userMediaFeeds.map((callFeed) => ( - - ))} - {isAdmin && showControls && ( - - )} -
-
- - {inviteModalState.isOpen && showControls && ( - - )} -
- ); -}; diff --git a/src/room/PTTFeed.module.css b/src/room/PTTFeed.module.css deleted file mode 100644 index 033fb43..0000000 --- a/src/room/PTTFeed.module.css +++ /dev/null @@ -1,19 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.audioFeed { - display: none; -} diff --git a/src/room/PTTFeed.tsx b/src/room/PTTFeed.tsx deleted file mode 100644 index 49a40e0..0000000 --- a/src/room/PTTFeed.tsx +++ /dev/null @@ -1,34 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; -import React from "react"; - -import { useCallFeed } from "../video-grid/useCallFeed"; -import { useMediaStream } from "../video-grid/useMediaStream"; -import styles from "./PTTFeed.module.css"; - -export function PTTFeed({ - callFeed, - audioOutputDevice, -}: { - callFeed: CallFeed; - audioOutputDevice: string; -}) { - const { isLocal, stream } = useCallFeed(callFeed); - const mediaRef = useMediaStream(stream, audioOutputDevice, isLocal); - return