Merge pull request #648 from vector-im/dbkr/tiles_for_everyone

Show tiles for members we're trying to connect to

public/locales/en-GB/app.json

@@ -2,6 +2,7 @@
   "{{count}} people connected|one": "{{count}} person connected",
   "{{count}} people connected|other": "{{count}} people connected",
   "{{displayName}}, your call is now ended": "{{displayName}}, your call is now ended",
+  "{{name}} (Connecting...)": "{{name}} (Connecting...)",
   "{{name}} is presenting": "{{name}} is presenting",
   "{{name}} is talking…": "{{name}} is talking…",
   "{{names}}, {{name}}": "{{names}}, {{name}}",

src/room/GroupCallView.tsx

@@ -76,7 +76,6 @@ export function GroupCallView({
     toggleScreensharing,
     requestingScreenshare,
     isScreensharing,
-    localScreenshareFeed,
     screenshareFeeds,
     participants,
     unencryptedEventsFromUsers,
@@ -221,6 +220,7 @@ export function GroupCallView({
           client={client}
           roomName={groupCall.room.name}
           avatarUrl={avatarUrl}
+          participants={participants}
           microphoneMuted={microphoneMuted}
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
@@ -230,7 +230,6 @@ export function GroupCallView({
           onLeave={onLeave}
           toggleScreensharing={toggleScreensharing}
           isScreensharing={isScreensharing}
-          localScreenshareFeed={localScreenshareFeed}
           screenshareFeeds={screenshareFeeds}
           roomIdOrAlias={roomIdOrAlias}
           unencryptedEventsFromUsers={unencryptedEventsFromUsers}

src/room/InCallView.tsx

@@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 interface Props {
   client: MatrixClient;
   groupCall: GroupCall;
+  participants: RoomMember[];
   roomName: string;
   avatarUrl: string;
   microphoneMuted: boolean;
@@ -82,14 +83,16 @@ interface Props {
   onLeave: () => void;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   roomIdOrAlias: string;
   unencryptedEventsFromUsers: Set<string>;
   hideHeader: boolean;
 }
 
-export interface Participant {
+// Represents something that should get a tile on the layout,
+// ie. a user's video feed or a screen share feed.
+export interface TileDescriptor {
   id: string;
+  member: RoomMember;
   focused: boolean;
   presenter: boolean;
   callFeed?: CallFeed;
@@ -99,6 +102,7 @@ export interface Participant {
 export function InCallView({
   client,
   groupCall,
+  participants,
   roomName,
   avatarUrl,
   microphoneMuted,
@@ -111,7 +115,6 @@ export function InCallView({
   toggleScreensharing,
   isScreensharing,
   screenshareFeeds,
-  localScreenshareFeed,
   roomIdOrAlias,
   unencryptedEventsFromUsers,
   hideHeader,
@@ -185,39 +188,48 @@ export function InCallView({
   }, [setLayout]);
 
   const items = useMemo(() => {
-    const participants: Participant[] = [];
+    const tileDescriptors: TileDescriptor[] = [];
 
-    for (const callFeed of userMediaFeeds) {
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
-        focused:
-          screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker,
-        isLocal: callFeed.isLocal(),
+    // one tile for each participants, to start with (we want a tile for everyone we
+    // think should be in the call, even if we don't have a media feed for them yet)
+    for (const p of participants) {
+      const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId);
+
+      // NB. this assumes that the same user can't join more than once from multiple
+      // devices, but the participants are just RoomMembers, so this assumption is baked
+      // into GroupCall itself.
+      tileDescriptors.push({
+        id: p.userId,
+        member: p,
+        callFeed: userMediaFeed,
+        focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
+        isLocal: p.userId === client.getUserId(),
         presenter: false,
       });
     }
 
-    for (const callFeed of screenshareFeeds) {
-      const userMediaItem = participants.find(
-        (item) => item.callFeed.userId === callFeed.userId
+    // add the screenshares too
+    for (const screenshareFeed of screenshareFeeds) {
+      const userMediaItem = tileDescriptors.find(
+        (item) => item.member.userId === screenshareFeed.userId
       );
 
       if (userMediaItem) {
         userMediaItem.presenter = true;
       }
 
-      participants.push({
-        id: callFeed.stream.id,
-        callFeed,
+      tileDescriptors.push({
+        id: screenshareFeed.stream.id,
+        member: userMediaItem?.member,
+        callFeed: screenshareFeed,
         focused: true,
-        isLocal: callFeed.isLocal(),
+        isLocal: screenshareFeed.isLocal(),
         presenter: false,
       });
     }
 
-    return participants;
-  }, [userMediaFeeds, activeSpeaker, screenshareFeeds]);
+    return tileDescriptors;
+  }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
 
   // The maximised participant: either the participant that the user has
   // manually put in fullscreen, or the focused (active) participant if the
@@ -281,7 +293,13 @@ export function InCallView({
 
     return (
       <VideoGrid items={items} layout={layout} disableAnimations={isSafari}>
-        {({ item, ...rest }: { item: Participant; [x: string]: unknown }) => (
+        {({
+          item,
+          ...rest
+        }: {
+          item: TileDescriptor;
+          [x: string]: unknown;
+        }) => (
           <VideoTileContainer
             key={item.id}
             item={item}

src/room/useGroupCall.ts

@@ -51,7 +51,6 @@ export interface UseGroupCallReturnType {
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   participants: RoomMember[];
   hasLocalParticipant: boolean;
@@ -68,7 +67,6 @@ interface State {
   microphoneMuted: boolean;
   localVideoMuted: boolean;
   screenshareFeeds: CallFeed[];
-  localScreenshareFeed: CallFeed;
   localDesktopCapturerSourceId: string;
   isScreensharing: boolean;
   requestingScreenshare: boolean;
@@ -89,7 +87,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       localVideoMuted,
       isScreensharing,
       screenshareFeeds,
-      localScreenshareFeed,
       localDesktopCapturerSourceId,
       participants,
       hasLocalParticipant,
@@ -107,7 +104,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     localVideoMuted: false,
     isScreensharing: false,
     screenshareFeeds: [],
-    localScreenshareFeed: null,
     localDesktopCapturerSourceId: null,
     requestingScreenshare: false,
     participants: [],
@@ -135,7 +131,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
         microphoneMuted: groupCall.isMicrophoneMuted(),
         localVideoMuted: groupCall.isLocalVideoMuted(),
         isScreensharing: groupCall.isScreensharing(),
-        localScreenshareFeed: groupCall.localScreenshareFeed,
         localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
         screenshareFeeds: [...groupCall.screenshareFeeds],
         participants: [...groupCall.participants],
@@ -172,12 +167,11 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
 
     function onLocalScreenshareStateChanged(
       isScreensharing: boolean,
-      localScreenshareFeed: CallFeed,
+      _localScreenshareFeed: CallFeed,
       localDesktopCapturerSourceId: string
     ): void {
       updateState({
         isScreensharing,
-        localScreenshareFeed,
         localDesktopCapturerSourceId,
       });
     }
@@ -228,7 +222,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
       microphoneMuted: groupCall.isMicrophoneMuted(),
       localVideoMuted: groupCall.isLocalVideoMuted(),
       isScreensharing: groupCall.isScreensharing(),
-      localScreenshareFeed: groupCall.localScreenshareFeed,
       localDesktopCapturerSourceId: groupCall.localDesktopCapturerSourceId,
       screenshareFeeds: [...groupCall.screenshareFeeds],
       participants: [...groupCall.participants],
@@ -412,7 +405,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
-    localScreenshareFeed,
     localDesktopCapturerSourceId,
     participants,
     hasLocalParticipant,

src/video-grid/AudioContainer.tsx

@@ -16,7 +16,7 @@ limitations under the License.
 
 import React, { FC, useEffect, useRef } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStreamTrackCount } from "./useMediaStream";
 
@@ -24,7 +24,7 @@ import { useMediaStreamTrackCount } from "./useMediaStream";
 // only way to a hook on an array
 
 interface AudioForParticipantProps {
-  item: Participant;
+  item: TileDescriptor;
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
@@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
 };
 
 interface AudioContainerProps {
-  items: Participant[];
+  items: TileDescriptor[];
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }

src/video-grid/VideoGrid.stories.tsx

@@ -16,11 +16,12 @@ limitations under the License.
 
 import React, { useState } from "react";
 import { useMemo } from "react";
+import { RoomMember } from "matrix-js-sdk";
 
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
   const { layout, setLayout } = useVideoGridLayout(false);
   const [participantCount, setParticipantCount] = useState(1);
 
-  const items: Participant[] = useMemo(
+  const items: TileDescriptor[] = useMemo(
     () =>
       new Array(participantCount).fill(undefined).map((_, i) => ({
         id: (i + 1).toString(),
+        member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
       })),
@@ -77,6 +79,7 @@ export const ParticipantsTest = () => {
               key={item.id}
               name={`User ${item.id}`}
               disableSpeakingIndicator={items.length < 3}
+              hasFeed={true}
               {...rest}
             />
           )}

src/video-grid/VideoGrid.tsx

@@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
 
 import styles from "./VideoGrid.module.css";
 import { Layout } from "../room/GridLayoutMenu";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface TilePosition {
   x: number;
@@ -36,7 +36,7 @@ interface TilePosition {
 interface Tile {
   key: Key;
   order: number;
-  item: Participant;
+  item: TileDescriptor;
   remove: boolean;
   focused: boolean;
   presenter: boolean;
@@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
   };
   width: number;
   height: number;
-  item: Participant;
+  item: TileDescriptor;
   [index: string]: unknown;
 }
 
 interface VideoGridProps {
-  items: Participant[];
+  items: TileDescriptor[];
   layout: Layout;
   disableAnimations?: boolean;
   children: (props: ChildrenProperties) => React.ReactNode;

src/video-grid/VideoTile.tsx

@@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
 
 interface Props {
   name: string;
+  hasFeed: Boolean;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
+      hasFeed,
       speaking,
       audioMuted,
       videoMuted,
@@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
+    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !videoMuted && <MicMutedIcon />}
               {videoMuted && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
+              <span title={caption}>{caption}</span>
             </div>
           ))}
         <video ref={mediaRef} playsInline disablePictureInPicture />

src/video-grid/VideoTileContainer.tsx

@@ -25,10 +25,10 @@ import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
 import { useModalTriggerState } from "../Modal";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface Props {
-  item: Participant;
+  item: TileDescriptor;
   width?: number;
   height?: number;
   getAvatar: (
@@ -41,7 +41,7 @@ interface Props {
   disableSpeakingIndicator: boolean;
   maximised: boolean;
   fullscreen: boolean;
-  onFullscreen: (item: Participant) => void;
+  onFullscreen: (item: TileDescriptor) => void;
 }
 
 export function VideoTileContainer({
@@ -65,9 +65,8 @@ export function VideoTileContainer({
     speaking,
     stream,
     purpose,
-    member,
   } = useCallFeed(item.callFeed);
-  const { rawDisplayName } = useRoomMemberName(member);
+  const { rawDisplayName } = useRoomMemberName(item.member);
   const [tileRef, mediaRef] = useSpatialMediaStream(
     stream,
     audioContext,
@@ -99,9 +98,10 @@ export function VideoTileContainer({
         videoMuted={videoMuted}
         screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
         name={rawDisplayName}
+        hasFeed={Boolean(item.callFeed)}
         ref={tileRef}
         mediaRef={mediaRef}
-        avatar={getAvatar && getAvatar(member, width, height)}
+        avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
         maximised={maximised}

src/video-grid/useCallFeed.ts

@@ -16,11 +16,10 @@ limitations under the License.
 
 import { useState, useEffect } from "react";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
-import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 
 interface CallFeedState {
-  member: RoomMember;
+  callFeed: CallFeed;
   isLocal: boolean;
   speaking: boolean;
   videoMuted: boolean;
@@ -32,7 +31,7 @@ interface CallFeedState {
 }
 function getCallFeedState(callFeed: CallFeed): CallFeedState {
   return {
-    member: callFeed ? callFeed.getMember() : null,
+    callFeed,
     isLocal: callFeed ? callFeed.isLocal() : false,
     speaking: callFeed ? callFeed.isSpeaking() : false,
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,

src/video-grid/useFullscreen.tsx

@@ -17,27 +17,27 @@ limitations under the License.
 
 import { useCallback, useEffect, useState } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useEventTarget } from "../useEvents";
 import { useCallFeed } from "./useCallFeed";
 
 export function useFullscreen(ref: React.RefObject<HTMLElement>): {
-  toggleFullscreen: (participant: Participant) => void;
-  fullscreenParticipant: Participant | null;
+  toggleFullscreen: (participant: TileDescriptor) => void;
+  fullscreenParticipant: TileDescriptor | null;
 } {
   const [fullscreenParticipant, setFullscreenParticipant] =
-    useState<Participant | null>(null);
+    useState<TileDescriptor | null>(null);
   const { disposed } = useCallFeed(fullscreenParticipant?.callFeed);
 
   const toggleFullscreen = useCallback(
-    (participant: Participant) => {
+    (tileDes: TileDescriptor) => {
       if (fullscreenParticipant) {
         document.exitFullscreen();
         setFullscreenParticipant(null);
       } else {
         try {
           ref.current.requestFullscreen();
-          setFullscreenParticipant(participant);
+          setFullscreenParticipant(tileDes);
         } catch (error) {
           console.warn("Failed to fullscreen:", error);
         }

src/video-grid/useMediaStream.ts

@@ -33,23 +33,33 @@ declare global {
 }
 
 export const useMediaStreamTrackCount = (
-  stream: MediaStream
+  stream: MediaStream | null
 ): [number, number] => {
+  const latestAudioTrackCount = stream ? stream.getAudioTracks().length : 0;
+  const latestVideoTrackCount = stream ? stream.getVideoTracks().length : 0;
+
   const [audioTrackCount, setAudioTrackCount] = useState(
-    stream.getAudioTracks().length
+    stream ? stream.getAudioTracks().length : 0
   );
   const [videoTrackCount, setVideoTrackCount] = useState(
-    stream.getVideoTracks().length
+    stream ? stream.getVideoTracks().length : 0
   );
 
   const tracksChanged = useCallback(() => {
-    setAudioTrackCount(stream.getAudioTracks().length);
-    setVideoTrackCount(stream.getVideoTracks().length);
+    setAudioTrackCount(stream ? stream.getAudioTracks().length : 0);
+    setVideoTrackCount(stream ? stream.getVideoTracks().length : 0);
   }, [stream]);
 
   useEventTarget(stream, "addtrack", tracksChanged);
   useEventTarget(stream, "removetrack", tracksChanged);
 
+  if (
+    latestAudioTrackCount !== audioTrackCount ||
+    latestVideoTrackCount !== videoTrackCount
+  ) {
+    tracksChanged();
+  }
+
   return [audioTrackCount, videoTrackCount];
 };