From 276684c1037bcef9e86c4ef3d8d1f02dba974f18 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 26 Jun 2023 13:48:41 -0400 Subject: [PATCH 01/16] Bring back fullscreen and picture-in-picture modes We're now using LiveKit's magic RoomAudioRenderer component to make sure everyone's audio is rendered regardless of whether they have a tile in the DOM. --- src/room/InCallView.tsx | 51 +++++++++++----- src/room/useFullscreen.ts | 114 +++++++++++++++++++++++++++++++++++ src/video-grid/VideoTile.tsx | 44 +++++++++++--- 3 files changed, 186 insertions(+), 23 deletions(-) create mode 100644 src/room/useFullscreen.ts diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index fd94aec..2ce3115 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -16,6 +16,8 @@ limitations under the License. import { ResizeObserver } from "@juggle/resize-observer"; import { + RoomAudioRenderer, + RoomContext, useLocalParticipant, useParticipants, useTracks, @@ -80,6 +82,7 @@ import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { VideoTile } from "../video-grid/VideoTile"; import { UserChoices, useLiveKit } from "../livekit/useLiveKit"; import { useMediaDevices } from "../livekit/useMediaDevices"; +import { useFullscreen } from "./useFullscreen"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -100,7 +103,13 @@ export function ActiveCall(props: ActiveCallProps) { userIdentity: `${props.client.getUserId()}:${props.client.getDeviceId()}`, }); - return livekitRoom && ; + return ( + livekitRoom && ( + + + + ) + ); } interface Props { @@ -172,9 +181,6 @@ export function InCallView({ const toggleCamera = useCallback(async () => { await localParticipant.setCameraEnabled(!isCameraEnabled); }, [localParticipant, isCameraEnabled]); - const toggleScreensharing = useCallback(async () => { - await localParticipant.setScreenShareEnabled(!isScreenShareEnabled); - }, [localParticipant, isScreenShareEnabled]); const joinRule = useJoinRule(groupCall.room); @@ -227,15 +233,19 @@ export function InCallView({ const noControls = reducedControls && bounds.height <= 400; const items = useParticipantTiles(livekitRoom, participants); + const { fullscreenItem, toggleFullscreen, exitFullscreen } = + useFullscreen(items); - // The maximised participant is the focused (active) participant, given the + // The maximised participant: either the participant that the user has + // manually put in fullscreen, or the focused (active) participant if the // window is too small to show everyone const maximisedParticipant = useMemo( () => - noControls - ? items.find((item) => item.focused) ?? items.at(0) ?? null - : null, - [noControls, items] + fullscreenItem ?? + (noControls + ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null + : null), + [fullscreenItem, noControls, items] ); const Grid = @@ -254,6 +264,9 @@ export function InCallView({ if (maximisedParticipant) { return ( {(props) => ( 2} showConnectionStats={showConnectionStats} {...props} @@ -321,6 +337,11 @@ export function InCallView({ [styles.maximised]: undefined, }); + const toggleScreensharing = useCallback(async () => { + exitFullscreen(); + await localParticipant.setScreenShareEnabled(!isScreenShareEnabled); + }, [localParticipant, isScreenShareEnabled, exitFullscreen]); + let footer: JSX.Element | null; if (noControls) { @@ -354,9 +375,7 @@ export function InCallView({ /> ); } - if (!maximisedParticipant) { - buttons.push(); - } + buttons.push(); } buttons.push( @@ -367,7 +386,7 @@ export function InCallView({ return (
- {!hideHeader && ( + {!hideHeader && maximisedParticipant === null && (
)}
+ {renderContent()} {footer}
@@ -469,6 +489,7 @@ function useParticipantTiles( local: sfuParticipant.isLocal, largeBaseSize: false, data: { + id, member, sfuParticipant, content: TileContent.UserMedia, @@ -478,14 +499,16 @@ function useParticipantTiles( // If there is a screen sharing enabled for this participant, create a tile for it as well. let screenShareTile: TileDescriptor | undefined; if (sfuParticipant.isScreenShareEnabled) { + const screenShareId = `${id}:screen-share`; screenShareTile = { ...userMediaTile, - id: `${id}:screen-share`, + id: screenShareId, focused: true, largeBaseSize: true, placeNear: id, data: { ...userMediaTile.data, + id: screenShareId, content: TileContent.ScreenShare, }, }; diff --git a/src/room/useFullscreen.ts b/src/room/useFullscreen.ts new file mode 100644 index 0000000..78ec1c8 --- /dev/null +++ b/src/room/useFullscreen.ts @@ -0,0 +1,114 @@ +/* +Copyright 2023 New Vector Ltd + + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "matrix-js-sdk/src/logger"; +import { useCallback, useLayoutEffect, useRef } from "react"; + +import { TileDescriptor } from "../video-grid/VideoGrid"; +import { useReactiveState } from "../useReactiveState"; +import { useEventTarget } from "../useEvents"; + +const isFullscreen = () => + Boolean(document.fullscreenElement) || + Boolean(document.webkitFullscreenElement); + +function enterFullscreen() { + if (document.body.requestFullscreen) { + document.body.requestFullscreen(); + } else if (document.body.webkitRequestFullscreen) { + document.body.webkitRequestFullscreen(); + } else { + logger.error("No available fullscreen API!"); + } +} + +function exitFullscreen() { + if (document.exitFullscreen) { + document.exitFullscreen(); + } else if (document.webkitExitFullscreen) { + document.webkitExitFullscreen(); + } else { + logger.error("No available fullscreen API!"); + } +} + +function useFullscreenChange(onFullscreenChange: () => void) { + useEventTarget(document.body, "fullscreenchange", onFullscreenChange); + useEventTarget(document.body, "webkitfullscreenchange", onFullscreenChange); +} + +/** + * Provides callbacks for controlling the full-screen view, which can hold one + * item at a time. + */ +export function useFullscreen(items: TileDescriptor[]): { + fullscreenItem: TileDescriptor | null; + toggleFullscreen: (itemId: string) => void; + exitFullscreen: () => void; +} { + const [fullscreenItem, setFullscreenItem] = + useReactiveState | null>( + (prevItem) => + prevItem == null + ? null + : items.find((i) => i.id === prevItem.id) ?? null, + [items] + ); + + const latestItems = useRef[]>(items); + latestItems.current = items; + + const latestFullscreenItem = useRef | null>(fullscreenItem); + latestFullscreenItem.current = fullscreenItem; + + const toggleFullscreen = useCallback( + (itemId: string) => { + setFullscreenItem( + latestFullscreenItem.current === null + ? latestItems.current.find((i) => i.id === itemId) ?? null + : null + ); + }, + [setFullscreenItem] + ); + + const exitFullscreenCallback = useCallback( + () => setFullscreenItem(null), + [setFullscreenItem] + ); + + useLayoutEffect(() => { + // Determine whether we need to change the fullscreen state + if (isFullscreen() !== (fullscreenItem !== null)) { + (fullscreenItem === null ? exitFullscreen : enterFullscreen)(); + } + }, [fullscreenItem]); + + // Detect when the user exits fullscreen through an external mechanism like + // browser chrome or the escape key + useFullscreenChange( + useCallback(() => { + if (!isFullscreen()) setFullscreenItem(null); + }, [setFullscreenItem]) + ); + + return { + fullscreenItem, + toggleFullscreen, + exitFullscreen: exitFullscreenCallback, + }; +} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 81a2361..1cf77d3 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React from "react"; +import React, { useCallback } from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; @@ -34,8 +34,10 @@ import styles from "./VideoTile.module.css"; import { ReactComponent as MicIcon } from "../icons/Mic.svg"; import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { useReactiveState } from "../useReactiveState"; +import { FullscreenButton } from "../button/Button"; export interface ItemData { + id: string; member?: RoomMember; sfuParticipant: LocalParticipant | RemoteParticipant; content: TileContent; @@ -48,7 +50,9 @@ export enum TileContent { interface Props { data: ItemData; - + maximised: boolean; + fullscreen: boolean; + onToggleFullscreen: (itemId: string) => void; // TODO: Refactor these props. targetWidth: number; targetHeight: number; @@ -62,6 +66,9 @@ export const VideoTile = React.forwardRef( ( { data, + maximised, + fullscreen, + onToggleFullscreen, className, style, targetWidth, @@ -93,17 +100,33 @@ export const VideoTile = React.forwardRef( } }, [member, setDisplayName]); - const audioEl = React.useRef(null); const { isMuted: microphoneMuted } = useMediaTrack( content === TileContent.UserMedia ? Track.Source.Microphone : Track.Source.ScreenShareAudio, - sfuParticipant, - { - element: audioEl, - } + sfuParticipant ); + const onFullscreen = useCallback(() => { + onToggleFullscreen(data.id); + }, [data, onToggleFullscreen]); + + const toolbarButtons: JSX.Element[] = []; + if (!sfuParticipant.isLocal) { + // TODO local volume option, which would also go here + + if (content === TileContent.ScreenShare) { + toolbarButtons.push( + + ); + } + } + // Firefox doesn't respect the disablePictureInPicture attribute // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831 @@ -117,11 +140,15 @@ export const VideoTile = React.forwardRef( showSpeakingIndicator, [styles.muted]: microphoneMuted, [styles.screenshare]: content === TileContent.ScreenShare, + [styles.maximised]: maximised, })} style={style} ref={tileRef} data-testid="videoTile" > + {toolbarButtons.length > 0 && (!maximised || fullscreen) && ( +
{toolbarButtons}
+ )} {content === TileContent.UserMedia && !sfuParticipant.isCameraEnabled && ( <>
@@ -134,7 +161,7 @@ export const VideoTile = React.forwardRef( /> )} - {content == TileContent.ScreenShare ? ( + {content === TileContent.ScreenShare ? (
{t("{{displayName}} is presenting", { displayName })}
@@ -155,7 +182,6 @@ export const VideoTile = React.forwardRef( : Track.Source.ScreenShare } /> -