xmj/element-call
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Show tiles for members we're trying to connect to

Browse source 
This should help give more context on what's going wrong in
splitbrain scenarios.

If users leave calls uncleanly, their tile will remain in until
their member event times out, which will be an hour from when they
joined the call. See https://github.com/vector-im/element-call/issues/639.

Part of https://github.com/vector-im/element-call/issues/616
This commit is contained in:
David Baker 2022-10-21 17:24:56 +01:00
parent 54fe2aa7a3
commit 1ea9432769
11 changed files with 85 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/room/GroupCallView.tsx
View file

@ -76,7 +76,6 @@ export function GroupCallView({
toggleScreensharing,
requestingScreenshare,
isScreensharing,
localScreenshareFeed,
screenshareFeeds,
participants,
unencryptedEventsFromUsers,
@ -221,6 +220,7 @@ export function GroupCallView({
client={client}
roomName={groupCall.room.name}
avatarUrl={avatarUrl}
participants={participants}
microphoneMuted={microphoneMuted}
localVideoMuted={localVideoMuted}
toggleLocalVideoMuted={toggleLocalVideoMuted}
@ -230,7 +230,6 @@ export function GroupCallView({
onLeave={onLeave}
toggleScreensharing={toggleScreensharing}
isScreensharing={isScreensharing}
localScreenshareFeed={localScreenshareFeed}
screenshareFeeds={screenshareFeeds}
roomIdOrAlias={roomIdOrAlias}
unencryptedEventsFromUsers={unencryptedEventsFromUsers}

60
src/room/InCallView.tsx
View file

@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
interface Props {
client: MatrixClient;
groupCall: GroupCall;
participants: RoomMember[];
roomName: string;
avatarUrl: string;
microphoneMuted: boolean;
@ -82,14 +83,16 @@ interface Props {
onLeave: () => void;
isScreensharing: boolean;
screenshareFeeds: CallFeed[];
localScreenshareFeed: CallFeed;
roomIdOrAlias: string;
unencryptedEventsFromUsers: Set<string>;
hideHeader: boolean;
}
export interface Participant {
// Represents something that should get a tile on the layout,
// ie. a user's video feed or a screen share feed.
export interface TileDescriptor {
id: string;
member: RoomMember;
focused: boolean;
presenter: boolean;
callFeed?: CallFeed;
@ -99,6 +102,7 @@ export interface Participant {
export function InCallView({
client,
groupCall,
participants,
roomName,
avatarUrl,
microphoneMuted,
@ -111,7 +115,6 @@ export function InCallView({
toggleScreensharing,
isScreensharing,
screenshareFeeds,
localScreenshareFeed,
roomIdOrAlias,
unencryptedEventsFromUsers,
hideHeader,
@ -185,39 +188,48 @@ export function InCallView({
}, [setLayout]);
const items = useMemo(() => {
const participants: Participant[] = [];
const tileDescriptors: TileDescriptor[] = [];
for (const callFeed of userMediaFeeds) {
participants.push({
id: callFeed.stream.id,
callFeed,
focused:
screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker,
isLocal: callFeed.isLocal(),
// one tile for each participants, to start with (we want a tile for everyone we
// think should be in the call, even if we don't have a media feed for them yet)
for (const p of participants) {
const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId);
// NB. this assumes that the same user can't join more than once from multiple
// devices, but the participants are just RoomMembers, so this assumption is baked
// into GroupCall itself.
tileDescriptors.push({
id: p.userId,
member: p,
callFeed: userMediaFeed,
focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
isLocal: p.userId === client.getUserId(),
presenter: false,
});
}
for (const callFeed of screenshareFeeds) {
const userMediaItem = participants.find(
(item) => item.callFeed.userId === callFeed.userId
// add the screenshares too
for (const screenshareFeed of screenshareFeeds) {
const userMediaItem = tileDescriptors.find(
(item) => item.member.userId === screenshareFeed.userId
);
if (userMediaItem) {
userMediaItem.presenter = true;
}
participants.push({
id: callFeed.stream.id,
callFeed,
tileDescriptors.push({
id: screenshareFeed.stream.id,
member: userMediaItem?.member,
callFeed: screenshareFeed,
focused: true,
isLocal: callFeed.isLocal(),
isLocal: screenshareFeed.isLocal(),
presenter: false,
});
}
return participants;
}, [userMediaFeeds, activeSpeaker, screenshareFeeds]);
return tileDescriptors;
}, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
// The maximised participant: either the participant that the user has
// manually put in fullscreen, or the focused (active) participant if the
@ -281,7 +293,13 @@ export function InCallView({
return (
<VideoGrid items={items} layout={layout} disableAnimations={isSafari}>
{({ item, ...rest }: { item: Participant; [x: string]: unknown }) => (
{({
item,
...rest
}: {
item: TileDescriptor;
[x: string]: unknown;
}) => (
<VideoTileContainer
key={item.id}
item={item}

10
src/room/useGroupCall.ts
View file

@ -49,7 +49,6 @@ export interface UseGroupCallReturnType {
requestingScreenshare: boolean;
isScreensharing: boolean;
screenshareFeeds: CallFeed[];
localScreenshareFeed: CallFeed;
localDesktopCapturerSourceId: string;
participants: RoomMember[];
hasLocalParticipant: boolean;
@ -66,7 +65,6 @@ interface State {
microphoneMuted: boolean;
localVideoMuted: boolean;
screenshareFeeds: CallFeed[];
localScreenshareFeed: CallFeed;
localDesktopCapturerSourceId: string;
isScreensharing: boolean;
requestingScreenshare: boolean;
@ -87,7 +85,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
localVideoMuted,
isScreensharing,
screenshareFeeds,
localScreenshareFeed,
localDesktopCapturerSourceId,
participants,
hasLocalParticipant,
@ -105,7 +102,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
localVideoMuted: false,
isScreensharing: false,
screenshareFeeds: [],
localScreenshareFeed: null,
localDesktopCapturerSourceId: null,
requestingScreenshare: false,
participants: [],
@ -133,7 +129,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
microphoneMuted: groupCall.isMicrophoneMuted(),
localVideoMuted: groupCall.isLocalVideoMuted(),
isScreensharing: groupCall.isScreensharing(),
localScreenshareFeed: groupCall.localScreenshareFeed,
localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
screenshareFeeds: [...groupCall.screenshareFeeds],
participants: [...groupCall.participants],
@ -170,12 +165,11 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
function onLocalScreenshareStateChanged(
isScreensharing: boolean,
localScreenshareFeed: CallFeed,
_localScreenshareFeed: CallFeed,
localDesktopCapturerSourceId: string
): void {
updateState({
isScreensharing,
localScreenshareFeed,
localDesktopCapturerSourceId,
});
}
@ -226,7 +220,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
microphoneMuted: groupCall.isMicrophoneMuted(),
localVideoMuted: groupCall.isLocalVideoMuted(),
isScreensharing: groupCall.isScreensharing(),
localScreenshareFeed: groupCall.localScreenshareFeed,
localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
screenshareFeeds: [...groupCall.screenshareFeeds],
participants: [...groupCall.participants],
@ -342,7 +335,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
requestingScreenshare,
isScreensharing,
screenshareFeeds,
localScreenshareFeed,
localDesktopCapturerSourceId,
participants,
hasLocalParticipant,

6
src/video-grid/AudioContainer.tsx
View file

@ -16,7 +16,7 @@ limitations under the License.
import React, { FC, useEffect, useRef } from "react";
import { Participant } from "../room/InCallView";
import { TileDescriptor } from "../room/InCallView";
import { useCallFeed } from "./useCallFeed";
import { useMediaStreamTrackCount } from "./useMediaStream";
@ -24,7 +24,7 @@ import { useMediaStreamTrackCount } from "./useMediaStream";
// only way to a hook on an array
interface AudioForParticipantProps {
item: Participant;
item: TileDescriptor;
audioContext: AudioContext;
audioDestination: AudioNode;
}
@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
};
interface AudioContainerProps {
items: Participant[];
items: TileDescriptor[];
audioContext: AudioContext;
audioDestination: AudioNode;
}

6
src/video-grid/VideoGrid.stories.tsx
View file

@ -16,11 +16,12 @@ limitations under the License.
import React, { useState } from "react";
import { useMemo } from "react";
import { RoomMember } from "matrix-js-sdk";
import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
import { VideoTile } from "./VideoTile";
import { Button } from "../button";
import { Participant } from "../room/InCallView";
import { TileDescriptor } from "../room/InCallView";
export default {
title: "VideoGrid",
@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
const { layout, setLayout } = useVideoGridLayout(false);
const [participantCount, setParticipantCount] = useState(1);
const items: Participant[] = useMemo(
const items: TileDescriptor[] = useMemo(
() =>
new Array(participantCount).fill(undefined).map((_, i) => ({
id: (i + 1).toString(),
member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
focused: false,
presenter: false,
})),

8
src/video-grid/VideoGrid.tsx
View file

@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
import styles from "./VideoGrid.module.css";
import { Layout } from "../room/GridLayoutMenu";
import { Participant } from "../room/InCallView";
import { TileDescriptor } from "../room/InCallView";
interface TilePosition {
x: number;
@ -36,7 +36,7 @@ interface TilePosition {
interface Tile {
key: Key;
order: number;
item: Participant;
item: TileDescriptor;
remove: boolean;
focused: boolean;
presenter: boolean;
@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
};
width: number;
height: number;
item: Participant;
item: TileDescriptor;
[index: string]: unknown;
}
interface VideoGridProps {
items: Participant[];
items: TileDescriptor[];
layout: Layout;
disableAnimations?: boolean;
children: (props: ChildrenProperties) => React.ReactNode;

6
src/video-grid/VideoTile.tsx
View file

@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
interface Props {
name: string;
hasFeed: Boolean;
speaking?: boolean;
audioMuted?: boolean;
videoMuted?: boolean;
@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
(
{
name,
hasFeed,
speaking,
audioMuted,
videoMuted,
@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
}
}
const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
return (
<animated.div
className={classNames(styles.videoTile, className, {
@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
<div className={classNames(styles.infoBubble, styles.memberName)}>
{audioMuted && !videoMuted && <MicMutedIcon />}
{videoMuted && <VideoMutedIcon />}
<span title={name}>{name}</span>
<span title={caption}>{caption}</span>
</div>
))}
<video ref={mediaRef} playsInline disablePictureInPicture />

12
src/video-grid/VideoTileContainer.tsx
View file

@ -25,10 +25,10 @@ import { useRoomMemberName } from "./useRoomMemberName";
import { VideoTile } from "./VideoTile";
import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
import { useModalTriggerState } from "../Modal";
import { Participant } from "../room/InCallView";
import { TileDescriptor } from "../room/InCallView";
interface Props {
item: Participant;
item: TileDescriptor;
width?: number;
height?: number;
getAvatar: (
@ -41,7 +41,7 @@ interface Props {
disableSpeakingIndicator: boolean;
maximised: boolean;
fullscreen: boolean;
onFullscreen: (item: Participant) => void;
onFullscreen: (item: TileDescriptor) => void;
}
export function VideoTileContainer({
@ -65,9 +65,8 @@ export function VideoTileContainer({
speaking,
stream,
purpose,
member,
} = useCallFeed(item.callFeed);
const { rawDisplayName } = useRoomMemberName(member);
const { rawDisplayName } = useRoomMemberName(item.member);
const [tileRef, mediaRef] = useSpatialMediaStream(
stream,
audioContext,
@ -99,9 +98,10 @@ export function VideoTileContainer({
videoMuted={videoMuted}
screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
name={rawDisplayName}
hasFeed={Boolean(item.callFeed)}
ref={tileRef}
mediaRef={mediaRef}
avatar={getAvatar && getAvatar(member, width, height)}
avatar={getAvatar && getAvatar(item.member, width, height)}
onOptionsPress={onOptionsPress}
localVolume={localVolume}
maximised={maximised}

5
src/video-grid/useCallFeed.ts
View file

@ -16,11 +16,10 @@ limitations under the License.
import { useState, useEffect } from "react";
import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
import { RoomMember } from "matrix-js-sdk/src/models/room-member";
import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
interface CallFeedState {
member: RoomMember;
callFeed: CallFeed;
isLocal: boolean;
speaking: boolean;
videoMuted: boolean;
@ -32,7 +31,7 @@ interface CallFeedState {
}
function getCallFeedState(callFeed: CallFeed): CallFeedState {
return {
member: callFeed ? callFeed.getMember() : null,
callFeed,
isLocal: callFeed ? callFeed.isLocal() : false,
speaking: callFeed ? callFeed.isSpeaking() : false,
videoMuted: callFeed ? callFeed.isVideoMuted() : true,

12
src/video-grid/useFullscreen.tsx
View file

@ -17,27 +17,27 @@ limitations under the License.
import { useCallback, useEffect, useState } from "react";
import { Participant } from "../room/InCallView";
import { TileDescriptor } from "../room/InCallView";
import { useEventTarget } from "../useEvents";
import { useCallFeed } from "./useCallFeed";
export function useFullscreen(ref: React.RefObject<HTMLElement>): {
toggleFullscreen: (participant: Participant) => void;
fullscreenParticipant: Participant | null;
toggleFullscreen: (participant: TileDescriptor) => void;
fullscreenParticipant: TileDescriptor | null;
} {
const [fullscreenParticipant, setFullscreenParticipant] =
useState<Participant | null>(null);
useState<TileDescriptor | null>(null);
const { disposed } = useCallFeed(fullscreenParticipant?.callFeed);
const toggleFullscreen = useCallback(
(participant: Participant) => {
(tileDes: TileDescriptor) => {
if (fullscreenParticipant) {
document.exitFullscreen();
setFullscreenParticipant(null);
} else {
try {
ref.current.requestFullscreen();
setFullscreenParticipant(participant);
setFullscreenParticipant(tileDes);
} catch (error) {
console.warn("Failed to fullscreen:", error);
}

18
src/video-grid/useMediaStream.ts
View file

@ -35,21 +35,31 @@ declare global {
export const useMediaStreamTrackCount = (
stream: MediaStream
): [number, number] => {
const latestAudioTrackCount = stream ? stream.getAudioTracks().length : 0;
const latestVideoTrackCount = stream ? stream.getVideoTracks().length : 0;
const [audioTrackCount, setAudioTrackCount] = useState(
stream.getAudioTracks().length
stream ? stream.getAudioTracks().length : 0
);
const [videoTrackCount, setVideoTrackCount] = useState(
stream.getVideoTracks().length
stream ? stream.getVideoTracks().length : 0
);
const tracksChanged = useCallback(() => {
setAudioTrackCount(stream.getAudioTracks().length);
setVideoTrackCount(stream.getVideoTracks().length);
setAudioTrackCount(stream ? stream.getAudioTracks().length : 0);
setVideoTrackCount(stream ? stream.getVideoTracks().length : 0);
}, [stream]);
useEventTarget(stream, "addtrack", tracksChanged);
useEventTarget(stream, "removetrack", tracksChanged);
if (
latestAudioTrackCount !== audioTrackCount ||
latestVideoTrackCount !== videoTrackCount
) {
tracksChanged();
}
return [audioTrackCount, videoTrackCount];
};