Show tiles for members we're trying to connect to

This should help give more context on what's going wrong in
splitbrain scenarios.

If users leave calls uncleanly, their tile will remain in until
their member event times out, which will be an hour from when they
joined the call. See https://github.com/vector-im/element-call/issues/639.

Part of https://github.com/vector-im/element-call/issues/616

src/room/GroupCallView.tsx

@@ -76,7 +76,6 @@ export function GroupCallView({
     toggleScreensharing,
     requestingScreenshare,
     isScreensharing,
-    localScreenshareFeed,
     screenshareFeeds,
     participants,
     unencryptedEventsFromUsers,
@@ -221,6 +220,7 @@ export function GroupCallView({
           client={client}
           roomName={groupCall.room.name}
           avatarUrl={avatarUrl}
+          participants={participants}
           microphoneMuted={microphoneMuted}
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
@@ -230,7 +230,6 @@ export function GroupCallView({
           onLeave={onLeave}
           toggleScreensharing={toggleScreensharing}
           isScreensharing={isScreensharing}
-          localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
           roomIdOrAlias={roomIdOrAlias}
           unencryptedEventsFromUsers={unencryptedEventsFromUsers}

src/room/InCallView.tsx

@@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 interface Props {
   client: MatrixClient;
   groupCall: GroupCall;
+  participants: RoomMember[];
   roomName: string;
   avatarUrl: string;
   microphoneMuted: boolean;
@@ -82,14 +83,16 @@ interface Props {
   onLeave: () => void;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   roomIdOrAlias: string;
   unencryptedEventsFromUsers: Set<string>;
   hideHeader: boolean;
 }
 
-export interface Participant {
+// Represents something that should get a tile on the layout,
+// ie. a user's video feed or a screen share feed.
+export interface TileDescriptor {
   id: string;
+  member: RoomMember;
   focused: boolean;
   presenter: boolean;
   callFeed?: CallFeed;
@@ -99,6 +102,7 @@ export interface Participant {
 export function InCallView({
   client,
   groupCall,
+  participants,
   roomName,
   avatarUrl,
   microphoneMuted,
@@ -111,7 +115,6 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  localScreenshareFeed,
   roomIdOrAlias,
   unencryptedEventsFromUsers,
   hideHeader,
@@ -185,39 +188,48 @@ export function InCallView({
   }, [setLayout]);
 
   const items = useMemo(() => {
-    const participants: Participant[] = [];
+    const tileDescriptors: TileDescriptor[] = [];
 
-    for (const callFeed of userMediaFeeds) {
+    // one tile for each participants, to start with (we want a tile for everyone we
-      participants.push({
+    // think should be in the call, even if we don't have a media feed for them yet)
-        id: callFeed.stream.id,
+    for (const p of participants) {
-        callFeed,
+      const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId);
-        focused:
+
-          screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker,
+      // NB. this assumes that the same user can't join more than once from multiple
-        isLocal: callFeed.isLocal(),
+      // devices, but the participants are just RoomMembers, so this assumption is baked
+      // into GroupCall itself.
+      tileDescriptors.push({
+        id: p.userId,
+        member: p,
+        callFeed: userMediaFeed,
+        focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
+        isLocal: p.userId === client.getUserId(),
         presenter: false,
       });
     }
 
-    for (const callFeed of screenshareFeeds) {
+    // add the screenshares too
-      const userMediaItem = participants.find(
+    for (const screenshareFeed of screenshareFeeds) {
-        (item) => item.callFeed.userId === callFeed.userId
+      const userMediaItem = tileDescriptors.find(
+        (item) => item.member.userId === screenshareFeed.userId
       );
 
       if (userMediaItem) {
         userMediaItem.presenter = true;
       }
 
-      participants.push({
+      tileDescriptors.push({
-        id: callFeed.stream.id,
+        id: screenshareFeed.stream.id,
-        callFeed,
+        member: userMediaItem?.member,
+        callFeed: screenshareFeed,
         focused: true,
-        isLocal: callFeed.isLocal(),
+        isLocal: screenshareFeed.isLocal(),
         presenter: false,
       });
     }
 
-    return participants;
+    return tileDescriptors;
-  }, [userMediaFeeds, activeSpeaker, screenshareFeeds]);
+  }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
 
   // The maximised participant: either the participant that the user has
   // manually put in fullscreen, or the focused (active) participant if the
@@ -281,7 +293,13 @@ export function InCallView({
 
     return (
       <VideoGrid items={items} layout={layout} disableAnimations={isSafari}>
-        {({ item, ...rest }: { item: Participant; [x: string]: unknown }) => (
+        {({
+          item,
+          ...rest
+        }: {
+          item: TileDescriptor;
+          [x: string]: unknown;
+        }) => (
           <VideoTileContainer
             key={item.id}
             item={item}

src/room/useGroupCall.ts

@@ -49,7 +49,6 @@ export interface UseGroupCallReturnType {
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   participants: RoomMember[];
   hasLocalParticipant: boolean;
@@ -66,7 +65,6 @@ interface State {
   microphoneMuted: boolean;
   localVideoMuted: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   isScreensharing: boolean;
   requestingScreenshare: boolean;
@@ -87,7 +85,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       localVideoMuted,
       isScreensharing,
       screenshareFeeds,
-      localScreenshareFeed,
       localDesktopCapturerSourceId,
       participants,
       hasLocalParticipant,
@@ -105,7 +102,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     localVideoMuted: false,
     isScreensharing: false,
     screenshareFeeds: [],
-    localScreenshareFeed: null,
     localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
@@ -133,7 +129,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
         microphoneMuted: groupCall.isMicrophoneMuted(),
         localVideoMuted: groupCall.isLocalVideoMuted(),
         isScreensharing: groupCall.isScreensharing(),
-        localScreenshareFeed: groupCall.localScreenshareFeed,
         localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
         screenshareFeeds: [...groupCall.screenshareFeeds],
         participants: [...groupCall.participants],
@@ -170,12 +165,11 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
 
     function onLocalScreenshareStateChanged(
       isScreensharing: boolean,
-      localScreenshareFeed: CallFeed,
+      _localScreenshareFeed: CallFeed,
       localDesktopCapturerSourceId: string
     ): void {
       updateState({
         isScreensharing,
-        localScreenshareFeed,
         localDesktopCapturerSourceId,
       });
     }
@@ -226,7 +220,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       microphoneMuted: groupCall.isMicrophoneMuted(),
       localVideoMuted: groupCall.isLocalVideoMuted(),
       isScreensharing: groupCall.isScreensharing(),
-      localScreenshareFeed: groupCall.localScreenshareFeed,
       localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
       screenshareFeeds: [...groupCall.screenshareFeeds],
       participants: [...groupCall.participants],
@@ -342,7 +335,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
-    localScreenshareFeed,
     localDesktopCapturerSourceId,
     participants,
     hasLocalParticipant,

src/video-grid/AudioContainer.tsx

@@ -16,7 +16,7 @@ limitations under the License.
 
 import React, { FC, useEffect, useRef } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStreamTrackCount } from "./useMediaStream";
 
@@ -24,7 +24,7 @@ import { useMediaStreamTrackCount } from "./useMediaStream";
 // only way to a hook on an array
 
 interface AudioForParticipantProps {
-  item: Participant;
+  item: TileDescriptor;
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
@@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
 };
 
 interface AudioContainerProps {
-  items: Participant[];
+  items: TileDescriptor[];
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }

src/video-grid/VideoGrid.stories.tsx

@@ -16,11 +16,12 @@ limitations under the License.
 
 import React, { useState } from "react";
 import { useMemo } from "react";
+import { RoomMember } from "matrix-js-sdk";
 
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
   const { layout, setLayout } = useVideoGridLayout(false);
   const [participantCount, setParticipantCount] = useState(1);
 
-  const items: Participant[] = useMemo(
+  const items: TileDescriptor[] = useMemo(
     () =>
       new Array(participantCount).fill(undefined).map((_, i) => ({
         id: (i + 1).toString(),
+        member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
       })),

src/video-grid/VideoGrid.tsx

@@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
 
 import styles from "./VideoGrid.module.css";
 import { Layout } from "../room/GridLayoutMenu";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface TilePosition {
   x: number;
@@ -36,7 +36,7 @@ interface TilePosition {
 interface Tile {
   key: Key;
   order: number;
-  item: Participant;
+  item: TileDescriptor;
   remove: boolean;
   focused: boolean;
   presenter: boolean;
@@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
   };
   width: number;
   height: number;
-  item: Participant;
+  item: TileDescriptor;
   [index: string]: unknown;
 }
 
 interface VideoGridProps {
-  items: Participant[];
+  items: TileDescriptor[];
   layout: Layout;
   disableAnimations?: boolean;
   children: (props: ChildrenProperties) => React.ReactNode;

src/video-grid/VideoTile.tsx

@@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
 
 interface Props {
   name: string;
+  hasFeed: Boolean;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
+      hasFeed,
       speaking,
       audioMuted,
       videoMuted,
@@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
+    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !videoMuted && <MicMutedIcon />}
               {videoMuted && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
+              <span title={caption}>{caption}</span>
             </div>
           ))}
         <video ref={mediaRef} playsInline disablePictureInPicture />

src/video-grid/VideoTileContainer.tsx

@@ -25,10 +25,10 @@ import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
 import { useModalTriggerState } from "../Modal";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface Props {
-  item: Participant;
+  item: TileDescriptor;
   width?: number;
   height?: number;
   getAvatar: (
@@ -41,7 +41,7 @@ interface Props {
   disableSpeakingIndicator: boolean;
   maximised: boolean;
   fullscreen: boolean;
-  onFullscreen: (item: Participant) => void;
+  onFullscreen: (item: TileDescriptor) => void;
 }
 
 export function VideoTileContainer({
@@ -65,9 +65,8 @@ export function VideoTileContainer({
     speaking,
     stream,
     purpose,
-    member,
   } = useCallFeed(item.callFeed);
-  const { rawDisplayName } = useRoomMemberName(member);
+  const { rawDisplayName } = useRoomMemberName(item.member);
   const [tileRef, mediaRef] = useSpatialMediaStream(
     stream,
     audioContext,
@@ -99,9 +98,10 @@ export function VideoTileContainer({
         videoMuted={videoMuted}
         screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
         name={rawDisplayName}
+        hasFeed={Boolean(item.callFeed)}
         ref={tileRef}
         mediaRef={mediaRef}
-        avatar={getAvatar && getAvatar(member, width, height)}
+        avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
         maximised={maximised}

src/video-grid/useCallFeed.ts

@@ -16,11 +16,10 @@ limitations under the License.
 
 import { useState, useEffect } from "react";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
-import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 
 interface CallFeedState {
-  member: RoomMember;
+  callFeed: CallFeed;
   isLocal: boolean;
   speaking: boolean;
   videoMuted: boolean;
@@ -32,7 +31,7 @@ interface CallFeedState {
 }
 function getCallFeedState(callFeed: CallFeed): CallFeedState {
   return {
-    member: callFeed ? callFeed.getMember() : null,
+    callFeed,
     isLocal: callFeed ? callFeed.isLocal() : false,
     speaking: callFeed ? callFeed.isSpeaking() : false,
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,

src/video-grid/useFullscreen.tsx

@@ -17,27 +17,27 @@ limitations under the License.
 
 import { useCallback, useEffect, useState } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useEventTarget } from "../useEvents";
 import { useCallFeed } from "./useCallFeed";
 
 export function useFullscreen(ref: React.RefObject<HTMLElement>): {
-  toggleFullscreen: (participant: Participant) => void;
+  toggleFullscreen: (participant: TileDescriptor) => void;
-  fullscreenParticipant: Participant | null;
+  fullscreenParticipant: TileDescriptor | null;
 } {
   const [fullscreenParticipant, setFullscreenParticipant] =
-    useState<Participant | null>(null);
+    useState<TileDescriptor | null>(null);
   const { disposed } = useCallFeed(fullscreenParticipant?.callFeed);
 
   const toggleFullscreen = useCallback(
-    (participant: Participant) => {
+    (tileDes: TileDescriptor) => {
       if (fullscreenParticipant) {
         document.exitFullscreen();
         setFullscreenParticipant(null);
       } else {
         try {
           ref.current.requestFullscreen();
-          setFullscreenParticipant(participant);
+          setFullscreenParticipant(tileDes);
         } catch (error) {
           console.warn("Failed to fullscreen:", error);
         }

src/video-grid/useMediaStream.ts

@@ -35,21 +35,31 @@ declare global {
 export const useMediaStreamTrackCount = (
   stream: MediaStream
 ): [number, number] => {
+  const latestAudioTrackCount = stream ? stream.getAudioTracks().length : 0;
+  const latestVideoTrackCount = stream ? stream.getVideoTracks().length : 0;
+
   const [audioTrackCount, setAudioTrackCount] = useState(
-    stream.getAudioTracks().length
+    stream ? stream.getAudioTracks().length : 0
   );
   const [videoTrackCount, setVideoTrackCount] = useState(
-    stream.getVideoTracks().length
+    stream ? stream.getVideoTracks().length : 0
   );
 
   const tracksChanged = useCallback(() => {
-    setAudioTrackCount(stream.getAudioTracks().length);
+    setAudioTrackCount(stream ? stream.getAudioTracks().length : 0);
-    setVideoTrackCount(stream.getVideoTracks().length);
+    setVideoTrackCount(stream ? stream.getVideoTracks().length : 0);
   }, [stream]);
 
   useEventTarget(stream, "addtrack", tracksChanged);
   useEventTarget(stream, "removetrack", tracksChanged);
 
+  if (
+    latestAudioTrackCount !== audioTrackCount ||
+    latestVideoTrackCount !== videoTrackCount
+  ) {
+    tracksChanged();
+  }
+
   return [audioTrackCount, videoTrackCount];
 };