Merge pull request #648 from vector-im/dbkr/tiles_for_everyone

Show tiles for members we're trying to connect to

src/video-grid/AudioContainer.tsx

@@ -16,7 +16,7 @@ limitations under the License.
 
 import React, { FC, useEffect, useRef } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStreamTrackCount } from "./useMediaStream";
 
@@ -24,7 +24,7 @@ import { useMediaStreamTrackCount } from "./useMediaStream";
 // only way to a hook on an array
 
 interface AudioForParticipantProps {
-  item: Participant;
+  item: TileDescriptor;
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }
@@ -78,7 +78,7 @@ export const AudioForParticipant: FC<AudioForParticipantProps> = ({
 };
 
 interface AudioContainerProps {
-  items: Participant[];
+  items: TileDescriptor[];
   audioContext: AudioContext;
   audioDestination: AudioNode;
 }

src/video-grid/VideoGrid.stories.tsx

@@ -16,11 +16,12 @@ limitations under the License.
 
 import React, { useState } from "react";
 import { useMemo } from "react";
+import { RoomMember } from "matrix-js-sdk";
 
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -33,10 +34,11 @@ export const ParticipantsTest = () => {
   const { layout, setLayout } = useVideoGridLayout(false);
   const [participantCount, setParticipantCount] = useState(1);
 
-  const items: Participant[] = useMemo(
+  const items: TileDescriptor[] = useMemo(
     () =>
       new Array(participantCount).fill(undefined).map((_, i) => ({
         id: (i + 1).toString(),
+        member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
       })),
@@ -77,6 +79,7 @@ export const ParticipantsTest = () => {
               key={item.id}
               name={`User ${item.id}`}
               disableSpeakingIndicator={items.length < 3}
+              hasFeed={true}
               {...rest}
             />
           )}

src/video-grid/VideoGrid.tsx

@@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ
 
 import styles from "./VideoGrid.module.css";
 import { Layout } from "../room/GridLayoutMenu";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface TilePosition {
   x: number;
@@ -36,7 +36,7 @@ interface TilePosition {
 interface Tile {
   key: Key;
   order: number;
-  item: Participant;
+  item: TileDescriptor;
   remove: boolean;
   focused: boolean;
   presenter: boolean;
@@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes {
   };
   width: number;
   height: number;
-  item: Participant;
+  item: TileDescriptor;
   [index: string]: unknown;
 }
 
 interface VideoGridProps {
-  items: Participant[];
+  items: TileDescriptor[];
   layout: Layout;
   disableAnimations?: boolean;
   children: (props: ChildrenProperties) => React.ReactNode;

src/video-grid/VideoTile.tsx

@@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button";
 
 interface Props {
   name: string;
+  hasFeed: Boolean;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -47,6 +48,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
+      hasFeed,
       speaking,
       audioMuted,
       videoMuted,
@@ -90,6 +92,8 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
+    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+
     return (
       <animated.div
         className={classNames(styles.videoTile, className, {
@@ -120,7 +124,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !videoMuted && <MicMutedIcon />}
               {videoMuted && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
+              <span title={caption}>{caption}</span>
             </div>
           ))}
         <video ref={mediaRef} playsInline disablePictureInPicture />

src/video-grid/VideoTileContainer.tsx

@@ -25,10 +25,10 @@ import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
 import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
 import { useModalTriggerState } from "../Modal";
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 
 interface Props {
-  item: Participant;
+  item: TileDescriptor;
   width?: number;
   height?: number;
   getAvatar: (
@@ -41,7 +41,7 @@ interface Props {
   disableSpeakingIndicator: boolean;
   maximised: boolean;
   fullscreen: boolean;
-  onFullscreen: (item: Participant) => void;
+  onFullscreen: (item: TileDescriptor) => void;
 }
 
 export function VideoTileContainer({
@@ -65,9 +65,8 @@ export function VideoTileContainer({
     speaking,
     stream,
     purpose,
-    member,
   } = useCallFeed(item.callFeed);
-  const { rawDisplayName } = useRoomMemberName(member);
+  const { rawDisplayName } = useRoomMemberName(item.member);
   const [tileRef, mediaRef] = useSpatialMediaStream(
     stream,
     audioContext,
@@ -99,9 +98,10 @@ export function VideoTileContainer({
         videoMuted={videoMuted}
         screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
         name={rawDisplayName}
+        hasFeed={Boolean(item.callFeed)}
         ref={tileRef}
         mediaRef={mediaRef}
-        avatar={getAvatar && getAvatar(member, width, height)}
+        avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
         maximised={maximised}

src/video-grid/useCallFeed.ts

@@ -16,11 +16,10 @@ limitations under the License.
 
 import { useState, useEffect } from "react";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
-import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 
 interface CallFeedState {
-  member: RoomMember;
+  callFeed: CallFeed;
   isLocal: boolean;
   speaking: boolean;
   videoMuted: boolean;
@@ -32,7 +31,7 @@ interface CallFeedState {
 }
 function getCallFeedState(callFeed: CallFeed): CallFeedState {
   return {
-    member: callFeed ? callFeed.getMember() : null,
+    callFeed,
     isLocal: callFeed ? callFeed.isLocal() : false,
     speaking: callFeed ? callFeed.isSpeaking() : false,
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,

src/video-grid/useFullscreen.tsx

@@ -17,27 +17,27 @@ limitations under the License.
 
 import { useCallback, useEffect, useState } from "react";
 
-import { Participant } from "../room/InCallView";
+import { TileDescriptor } from "../room/InCallView";
 import { useEventTarget } from "../useEvents";
 import { useCallFeed } from "./useCallFeed";
 
 export function useFullscreen(ref: React.RefObject<HTMLElement>): {
-  toggleFullscreen: (participant: Participant) => void;
-  fullscreenParticipant: Participant | null;
+  toggleFullscreen: (participant: TileDescriptor) => void;
+  fullscreenParticipant: TileDescriptor | null;
 } {
   const [fullscreenParticipant, setFullscreenParticipant] =
-    useState<Participant | null>(null);
+    useState<TileDescriptor | null>(null);
   const { disposed } = useCallFeed(fullscreenParticipant?.callFeed);
 
   const toggleFullscreen = useCallback(
-    (participant: Participant) => {
+    (tileDes: TileDescriptor) => {
       if (fullscreenParticipant) {
         document.exitFullscreen();
         setFullscreenParticipant(null);
       } else {
         try {
           ref.current.requestFullscreen();
-          setFullscreenParticipant(participant);
+          setFullscreenParticipant(tileDes);
         } catch (error) {
           console.warn("Failed to fullscreen:", error);
         }

src/video-grid/useMediaStream.ts

@@ -33,23 +33,33 @@ declare global {
 }
 
 export const useMediaStreamTrackCount = (
-  stream: MediaStream
+  stream: MediaStream | null
 ): [number, number] => {
+  const latestAudioTrackCount = stream ? stream.getAudioTracks().length : 0;
+  const latestVideoTrackCount = stream ? stream.getVideoTracks().length : 0;
+
   const [audioTrackCount, setAudioTrackCount] = useState(
-    stream.getAudioTracks().length
+    stream ? stream.getAudioTracks().length : 0
   );
   const [videoTrackCount, setVideoTrackCount] = useState(
-    stream.getVideoTracks().length
+    stream ? stream.getVideoTracks().length : 0
   );
 
   const tracksChanged = useCallback(() => {
-    setAudioTrackCount(stream.getAudioTracks().length);
-    setVideoTrackCount(stream.getVideoTracks().length);
+    setAudioTrackCount(stream ? stream.getAudioTracks().length : 0);
+    setVideoTrackCount(stream ? stream.getVideoTracks().length : 0);
   }, [stream]);
 
   useEventTarget(stream, "addtrack", tracksChanged);
   useEventTarget(stream, "removetrack", tracksChanged);
 
+  if (
+    latestAudioTrackCount !== audioTrackCount ||
+    latestVideoTrackCount !== videoTrackCount
+  ) {
+    tracksChanged();
+  }
+
   return [audioTrackCount, videoTrackCount];
 };