xmj/element-call
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Simplify overly complicated VideoGrid

Browse source
This commit is contained in:
Daniel Abramov 2023-06-05 20:51:01 +02:00
parent b11ab01bbe
commit 79018606b2
4 changed files with 139 additions and 149 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

176
src/room/InCallView.tsx
View file

@ -14,17 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useEffect, useCallback, useMemo, useRef } from "react";
import { usePreventScroll } from "@react-aria/overlays";
import useMeasure from "react-use-measure";
import { ResizeObserver } from "@juggle/resize-observer";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
import classNames from "classnames";
import { useTranslation } from "react-i18next";
import { JoinRule } from "matrix-js-sdk/src/@types/partials";
import { Room, Track } from "livekit-client";
import {
useLiveKitRoom,
useLocalParticipant,
@ -32,15 +22,19 @@ import {
useToken,
useTracks,
} from "@livekit/components-react";
import { usePreventScroll } from "@react-aria/overlays";
import classNames from "classnames";
import { Room, Track } from "livekit-client";
import { JoinRule } from "matrix-js-sdk/src/@types/partials";
import { MatrixClient } from "matrix-js-sdk/src/client";
import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
import React, { useCallback, useEffect, useMemo, useRef } from "react";
import { useTranslation } from "react-i18next";
import useMeasure from "react-use-measure";
import type { IWidgetApiRequest } from "matrix-widget-api";
import styles from "./InCallView.module.css";
import {
HangupButton,
MicButton,
VideoButton,
ScreenshareButton,
} from "../button";
import { Avatar } from "../Avatar";
import {
Header,
LeftNav,
@ -48,27 +42,36 @@ import {
RoomHeaderInfo,
VersionMismatchWarning,
} from "../Header";
import { VideoGrid, useVideoGridLayout } from "../video-grid/VideoGrid";
import { VideoTileContainer } from "../video-grid/VideoTileContainer";
import { GroupCallInspector } from "./GroupCallInspector";
import { OverflowMenu } from "./OverflowMenu";
import { GridLayoutMenu } from "./GridLayoutMenu";
import { Avatar } from "../Avatar";
import { UserMenuContainer } from "../UserMenuContainer";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
import { PosthogAnalytics } from "../PosthogAnalytics";
import { widget, ElementWidgetActions } from "../widget";
import { useJoinRule } from "./useJoinRule";
import { useUrlParams } from "../UrlParams";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { ParticipantInfo } from "./useGroupCall";
import { TileDescriptor } from "../video-grid/TileDescriptor";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { UserMenuContainer } from "../UserMenuContainer";
import {
HangupButton,
MicButton,
ScreenshareButton,
VideoButton,
} from "../button";
import { MediaDevicesState } from "../settings/mediaDevices";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { useShowInspector } from "../settings/useSetting";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import {
TileDescriptor,
VideoGrid,
useVideoGridLayout,
} from "../video-grid/VideoGrid";
import { ItemData, VideoTileContainer } from "../video-grid/VideoTileContainer";
import { ElementWidgetActions, widget } from "../widget";
import { GridLayoutMenu } from "./GridLayoutMenu";
import { GroupCallInspector } from "./GroupCallInspector";
import styles from "./InCallView.module.css";
import { OverflowMenu } from "./OverflowMenu";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { MatrixInfo } from "./VideoPreview";
import { useJoinRule } from "./useJoinRule";
import { ParticipantInfo } from "./useGroupCall";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
// There is currently a bug in Safari our our code with cloning and sending MediaStreams
@ -228,46 +231,14 @@ export function InCallView({
}
}, [setLayout]);
const sfuParticipants = useParticipants({
room: livekitRoom,
});
const items = useMemo(() => {
const localUserId = client.getUserId()!;
const localDeviceId = client.getDeviceId()!;
// One tile for each participant, to start with (we want a tile for everyone we
// think should be in the call, even if we don't have a call feed for them yet)
const tileDescriptors: TileDescriptor[] = [];
for (const [member, participantMap] of participants) {
for (const [deviceId, { presenter }] of participantMap) {
const id = `${member.userId}:${deviceId}`;
const sfuParticipant = sfuParticipants.find((p) => p.identity === id);
const hasScreenShare =
sfuParticipant?.getTrack(Track.Source.ScreenShare) !== undefined;
tileDescriptors.push({
id,
member,
focused: hasScreenShare && !sfuParticipant?.isLocal,
isLocal: member.userId == localUserId && deviceId == localDeviceId,
presenter,
sfuParticipant,
});
}
}
PosthogAnalytics.instance.eventCallEnded.cacheParticipantCountChanged(
tileDescriptors.length
);
return tileDescriptors;
}, [client, participants, sfuParticipants]);
const reducedControls = boundsValid && bounds.width <= 400;
const noControls = reducedControls && bounds.height <= 400;
const items = useParticipantTiles(livekitRoom, participants, {
userId: client.getUserId()!,
deviceId: client.getDeviceId()!,
});
// The maximised participant: the focused (active) participant if the
// window is too small to show everyone.
const maximisedParticipant = useMemo(
@ -309,7 +280,7 @@ export function InCallView({
height={bounds.height}
width={bounds.width}
key={maximisedParticipant.id}
item={maximisedParticipant}
item={maximisedParticipant.data}
getAvatar={renderAvatar}
maximised={Boolean(maximisedParticipant)}
/>
@ -322,19 +293,12 @@ export function InCallView({
layout={layout}
disableAnimations={prefersReducedMotion || isSafari}
>
{({
item,
...rest
}: {
item: TileDescriptor;
[x: string]: unknown;
}) => (
{(child) => (
<VideoTileContainer
key={item.id}
item={item}
getAvatar={renderAvatar}
maximised={false}
{...rest}
item={child.data}
{...child}
/>
)}
</VideoGrid>
@ -424,3 +388,53 @@ export function InCallView({
</div>
);
}
interface ParticipantID {
userId: string;
deviceId: string;
}
function useParticipantTiles(
livekitRoom: Room,
participants: Map<RoomMember, Map<string, ParticipantInfo>>,
local: ParticipantID
): TileDescriptor<ItemData>[] {
const sfuParticipants = useParticipants({
room: livekitRoom,
});
const [localUserId, localDeviceId] = [local.userId, local.deviceId];
const items = useMemo(() => {
const tiles: TileDescriptor<ItemData>[] = [];
for (const [member, participantMap] of participants) {
for (const [deviceId] of participantMap) {
const id = `${member.userId}:${deviceId}`;
const sfuParticipant = sfuParticipants.find((p) => p.identity === id);
const hasScreenShare =
sfuParticipant?.getTrack(Track.Source.ScreenShare) !== undefined;
const descriptor = {
id,
focused: hasScreenShare && !sfuParticipant?.isLocal,
local: member.userId == localUserId && deviceId == localDeviceId,
data: {
member,
sfuParticipant,
},
};
tiles.push(descriptor);
}
}
PosthogAnalytics.instance.eventCallEnded.cacheParticipantCountChanged(
tiles.length
);
return tiles;
}, [localUserId, localDeviceId, participants, sfuParticipants]);
return items;
}

29
src/video-grid/TileDescriptor.tsx
View file

@ -1,29 +0,0 @@
/*
Copyright 2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { RoomMember } from "matrix-js-sdk";
import { LocalParticipant, RemoteParticipant } from "livekit-client";
// Represents something that should get a tile on the layout,
// ie. a user's video feed or a screen share feed.
export interface TileDescriptor {
id: string;
member: RoomMember;
focused: boolean;
presenter: boolean;
isLocal?: boolean;
sfuParticipant?: LocalParticipant | RemoteParticipant;
}

74
src/video-grid/VideoGrid.tsx
View file

@ -23,7 +23,6 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
import styles from "./VideoGrid.module.css";
import { Layout } from "../room/GridLayoutMenu";
import { TileDescriptor } from "./TileDescriptor";
interface TilePosition {
x: number;
@ -33,13 +32,12 @@ interface TilePosition {
zIndex: number;
}
interface Tile {
interface Tile<T> {
key: Key;
order: number;
item: TileDescriptor;
item: TileDescriptor<T>;
remove: boolean;
focused: boolean;
presenter: boolean;
}
type LayoutDirection = "vertical" | "horizontal";
@ -112,7 +110,6 @@ const getPipGap = (gridAspectRatio: number, gridWidth: number): number =>
function getTilePositions(
tileCount: number,
focusedTileCount: number,
hasPresenter: boolean,
gridWidth: number,
gridHeight: number,
pipXRatio: number,
@ -120,7 +117,7 @@ function getTilePositions(
layout: Layout
): TilePosition[] {
if (layout === "freedom") {
if (tileCount === 2 && !hasPresenter && focusedTileCount === 0) {
if (tileCount === 2 && focusedTileCount === 0) {
return getOneOnOneLayoutTilePositions(
gridWidth,
gridHeight,
@ -654,7 +651,7 @@ function getSubGridPositions(
// Sets the 'order' property on tiles based on the layout param and
// other properties of the tiles, eg. 'focused' and 'presenter'
function reorderTiles(tiles: Tile[], layout: Layout) {
function reorderTiles<T>(tiles: Tile<T>[], layout: Layout) {
// We use a special layout for 1:1 to always put the local tile first.
// We only do this if there are two tiles (obviously) and exactly one
// of them is local: during startup we can have tiles from other users
@ -664,16 +661,16 @@ function reorderTiles(tiles: Tile[], layout: Layout) {
if (
layout === "freedom" &&
tiles.length === 2 &&
tiles.filter((t) => t.item.isLocal).length === 1 &&
!tiles.some((t) => t.presenter || t.focused)
tiles.filter((t) => t.item.local).length === 1 &&
!tiles.some((t) => t.focused)
) {
// 1:1 layout
tiles.forEach((tile) => (tile.order = tile.item.isLocal ? 0 : 1));
tiles.forEach((tile) => (tile.order = tile.item.local ? 0 : 1));
} else {
const focusedTiles: Tile[] = [];
const otherTiles: Tile[] = [];
const focusedTiles: Tile<T>[] = [];
const otherTiles: Tile<T>[] = [];
const orderedTiles: Tile[] = new Array(tiles.length);
const orderedTiles: Tile<T>[] = new Array(tiles.length);
tiles.forEach((tile) => (orderedTiles[tile.order] = tile));
orderedTiles.forEach((tile) =>
@ -692,8 +689,9 @@ interface DragTileData {
y: number;
}
interface ChildrenProperties extends ReactDOMAttributes {
interface ChildrenProperties<T> extends ReactDOMAttributes {
key: Key;
data: T;
style: {
scale: SpringValue<number>;
opacity: SpringValue<number>;
@ -701,29 +699,36 @@ interface ChildrenProperties extends ReactDOMAttributes {
};
width: number;
height: number;
item: TileDescriptor;
[index: string]: unknown;
}
interface VideoGridProps {
items: TileDescriptor[];
interface VideoGridProps<T> {
items: TileDescriptor<T>[];
layout: Layout;
disableAnimations?: boolean;
children: (props: ChildrenProperties) => React.ReactNode;
disableAnimations: boolean;
children: (props: ChildrenProperties<T>) => React.ReactNode;
}
export function VideoGrid({
// Represents something that should get a tile on the layout,
// ie. a user's video feed or a screen share feed.
export interface TileDescriptor<T> {
id: string;
focused: boolean;
local: boolean;
data: T;
}
export function VideoGrid<T>({
items,
layout,
disableAnimations,
children,
}: VideoGridProps) {
children: createChild,
}: VideoGridProps<T>) {
// Place the PiP in the bottom right corner by default
const [pipXRatio, setPipXRatio] = useState(1);
const [pipYRatio, setPipYRatio] = useState(1);
const [{ tiles, tilePositions }, setTileState] = useState<{
tiles: Tile[];
tiles: Tile<T>[];
tilePositions: TilePosition[];
}>({
tiles: [],
@ -739,7 +744,7 @@ export function VideoGrid({
useEffect(() => {
setTileState(({ tiles, ...rest }) => {
const newTiles: Tile[] = [];
const newTiles: Tile<T>[] = [];
const removedTileKeys: Set<Key> = new Set();
for (const tile of tiles) {
@ -766,7 +771,6 @@ export function VideoGrid({
item,
remove,
focused,
presenter: item.presenter,
});
}
@ -781,13 +785,12 @@ export function VideoGrid({
continue;
}
const newTile: Tile = {
const newTile: Tile<T> = {
key: item.id,
order: existingTile?.order ?? newTiles.length,
item,
remove: false,
focused: layout === "spotlight" && item.focused,
presenter: item.presenter,
};
if (existingTile) {
@ -808,7 +811,7 @@ export function VideoGrid({
}
setTileState(({ tiles, ...rest }) => {
const newTiles: Tile[] = tiles
const newTiles: Tile<T>[] = tiles
.filter((tile) => !removedTileKeys.has(tile.key))
.map((tile) => ({ ...tile })); // clone before reordering
reorderTiles(newTiles, layout);
@ -824,7 +827,6 @@ export function VideoGrid({
tilePositions: getTilePositions(
newTiles.length,
focusedTileCount,
newTiles.some((t) => t.presenter),
gridBounds.width,
gridBounds.height,
pipXRatio,
@ -849,7 +851,6 @@ export function VideoGrid({
tilePositions: getTilePositions(
newTiles.length,
focusedTileCount,
newTiles.some((t) => t.presenter),
gridBounds.width,
gridBounds.height,
pipXRatio,
@ -863,7 +864,7 @@ export function VideoGrid({
const tilePositionsValid = useRef(false);
const animate = useCallback(
(tiles: Tile[]) => {
(tiles: Tile<T>[]) => {
// Whether the tile positions were valid at the time of the previous
// animation
const tilePositionsWereValid = tilePositionsValid.current;
@ -1005,7 +1006,6 @@ export function VideoGrid({
tilePositions: getTilePositions(
newTiles.length,
focusedTileCount,
newTiles.some((t) => t.presenter),
gridBounds.width,
gridBounds.height,
pipXRatio,
@ -1037,9 +1037,9 @@ export function VideoGrid({
let newTiles = tiles;
if (tiles.length === 2 && !tiles.some((t) => t.presenter || t.focused)) {
if (tiles.length === 2 && !tiles.some((t) => t.focused)) {
// We're in 1:1 mode, so only the local tile should be draggable
if (!dragTile.item.isLocal) return;
if (!dragTile.item.local) return;
// Position should only update on the very last event, to avoid
// compounding the offset on every drag event
@ -1179,9 +1179,10 @@ export function VideoGrid({
const tile = tiles[i];
const tilePosition = tilePositions[tile.order];
return children({
return createChild({
...bindTile(tile.key),
key: tile.key,
data: tile.item.data,
style: {
boxShadow: shadow.to(
(s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px`
@ -1190,7 +1191,6 @@ export function VideoGrid({
},
width: tilePosition.width,
height: tilePosition.height,
item: tile.item,
});
})}
</div>

9
src/video-grid/VideoTileContainer.tsx
View file

@ -16,13 +16,18 @@ limitations under the License.
import React from "react";
import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { LocalParticipant, RemoteParticipant } from "livekit-client";
import { useRoomMemberName } from "./useRoomMemberName";
import { VideoTile } from "./VideoTile";
import { TileDescriptor } from "./TileDescriptor";
export interface ItemData {
member: RoomMember;
sfuParticipant?: LocalParticipant | RemoteParticipant;
}
interface Props {
item: TileDescriptor;
item: ItemData;
width?: number;
height?: number;
getAvatar: (