Merge pull request #752 from vector-im/dbkr/waiting_for_media

Add a 'waiting for video' state to media tiles

package.json

@@ -45,7 +45,7 @@
     "i18next": "^21.10.0",
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c6ee258789c9e01d328b5d9158b5b372e3a0da82",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3f1c3392d45b0fc054c3788cc6c043cd5b4fb730",
     "matrix-widget-api": "^1.0.0",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",

public/locales/en-GB/app.json

@@ -3,6 +3,7 @@
   "{{count}} people connected|other": "{{count}} people connected",
   "{{displayName}}, your call is now ended": "{{displayName}}, your call is now ended",
   "{{name}} (Connecting...)": "{{name}} (Connecting...)",
+  "{{name}} (Waiting for video...)": "{{name}} (Waiting for video...)",
   "{{name}} is presenting": "{{name}} is presenting",
   "{{name}} is talking…": "{{name}} is talking…",
   "{{names}}, {{name}}": "{{names}}, {{name}}",

src/room/GroupCallView.tsx

@@ -79,6 +79,7 @@ export function GroupCallView({
     isScreensharing,
     screenshareFeeds,
     participants,
+    calls,
     unencryptedEventsFromUsers,
   } = useGroupCall(groupCall);
 
@@ -235,6 +236,7 @@ export function GroupCallView({
           roomName={groupCall.room.name}
           avatarUrl={avatarUrl}
           participants={participants}
+          calls={calls}
           microphoneMuted={microphoneMuted}
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}

src/room/InCallView.tsx

@@ -14,7 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useEffect, useCallback, useMemo, useRef } from "react";
+import React, {
+  useEffect,
+  useCallback,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
 import { usePreventScroll } from "@react-aria/overlays";
 import useMeasure from "react-use-measure";
 import { ResizeObserver } from "@juggle/resize-observer";
@@ -25,6 +31,11 @@ import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
 import classNames from "classnames";
 import { useTranslation } from "react-i18next";
 import { JoinRule } from "matrix-js-sdk/src/@types/partials";
+import {
+  CallEvent,
+  CallState,
+  MatrixCall,
+} from "matrix-js-sdk/src/webrtc/call";
 
 import type { IWidgetApiRequest } from "matrix-widget-api";
 import styles from "./InCallView.module.css";
@@ -73,6 +84,7 @@ interface Props {
   client: MatrixClient;
   groupCall: GroupCall;
   participants: RoomMember[];
+  calls: MatrixCall[];
   roomName: string;
   avatarUrl: string;
   microphoneMuted: boolean;
@@ -90,6 +102,12 @@ interface Props {
   hideHeader: boolean;
 }
 
+export enum ConnectionState {
+  EstablishingCall = "establishing call", // call hasn't been established yet
+  WaitMedia = "wait_media", // call is set up, waiting for ICE to connect
+  Connected = "connected", // media is flowing
+}
+
 // Represents something that should get a tile on the layout,
 // ie. a user's video feed or a screen share feed.
 export interface TileDescriptor {
@@ -99,12 +117,14 @@ export interface TileDescriptor {
   presenter: boolean;
   callFeed?: CallFeed;
   isLocal?: boolean;
+  connectionState: ConnectionState;
 }
 
 export function InCallView({
   client,
   groupCall,
   participants,
+  calls,
   roomName,
   avatarUrl,
   microphoneMuted,
@@ -154,6 +174,50 @@ export function InCallView({
 
   const { hideScreensharing } = useUrlParams();
 
+  const makeConnectionStatesMap = useCallback(() => {
+    const newConnStates = new Map<string, ConnectionState>();
+    for (const participant of participants) {
+      const userCall = groupCall.getCallByUserId(participant.userId);
+      const feed = userMediaFeeds.find((f) => f.userId === participant.userId);
+      let connectionState = ConnectionState.EstablishingCall;
+      if (feed && feed.isLocal()) {
+        connectionState = ConnectionState.Connected;
+      } else if (userCall) {
+        if (userCall.state === CallState.Connected) {
+          connectionState = ConnectionState.Connected;
+        } else if (userCall.state === CallState.Connecting) {
+          connectionState = ConnectionState.WaitMedia;
+        }
+      }
+      newConnStates.set(participant.userId, connectionState);
+    }
+    return newConnStates;
+  }, [groupCall, participants, userMediaFeeds]);
+
+  const [connStates, setConnStates] = useState(
+    new Map<string, ConnectionState>()
+  );
+
+  const updateConnectionStates = useCallback(() => {
+    setConnStates(makeConnectionStatesMap());
+  }, [setConnStates, makeConnectionStatesMap]);
+
+  useEffect(() => {
+    for (const call of calls) {
+      call.on(CallEvent.State, updateConnectionStates);
+    }
+
+    return () => {
+      for (const call of calls) {
+        call.off(CallEvent.State, updateConnectionStates);
+      }
+    };
+  }, [calls, updateConnectionStates]);
+
+  useEffect(() => {
+    updateConnectionStates();
+  }, [participants, updateConnectionStates]);
+
   useEffect(() => {
     widget?.api.transport.send(
       layout === "freedom"
@@ -208,6 +272,7 @@ export function InCallView({
         focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker,
         isLocal: p.userId === client.getUserId(),
         presenter: false,
+        connectionState: connStates.get(p.userId),
       });
     }
 
@@ -231,11 +296,19 @@ export function InCallView({
         focused: true,
         isLocal: screenshareFeed.isLocal(),
         presenter: false,
+        connectionState: connStates.get(screenshareFeed.userId),
       });
     }
 
     return tileDescriptors;
-  }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]);
+  }, [
+    client,
+    participants,
+    userMediaFeeds,
+    activeSpeaker,
+    screenshareFeeds,
+    connStates,
+  ]);
 
   // The maximised participant: either the participant that the user has
   // manually put in fullscreen, or the focused (active) participant if the

src/video-grid/VideoGrid.stories.tsx

@@ -21,7 +21,7 @@ import { RoomMember } from "matrix-js-sdk";
 import { VideoGrid, useVideoGridLayout } from "./VideoGrid";
 import { VideoTile } from "./VideoTile";
 import { Button } from "../button";
-import { TileDescriptor } from "../room/InCallView";
+import { ConnectionState, TileDescriptor } from "../room/InCallView";
 
 export default {
   title: "VideoGrid",
@@ -41,6 +41,7 @@ export const ParticipantsTest = () => {
         member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`),
         focused: false,
         presenter: false,
+        connectionState: ConnectionState.Connected,
       })),
     [participantCount]
   );
@@ -79,7 +80,7 @@ export const ParticipantsTest = () => {
               key={item.id}
               name={`User ${item.id}`}
               disableSpeakingIndicator={items.length < 3}
-              hasFeed={true}
+              connectionState={ConnectionState.Connected}
               {...rest}
             />
           )}

src/video-grid/VideoTile.tsx

@@ -23,10 +23,11 @@ import styles from "./VideoTile.module.css";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
 import { AudioButton, FullscreenButton } from "../button/Button";
+import { ConnectionState } from "../room/InCallView";
 
 interface Props {
   name: string;
-  hasFeed: Boolean;
+  connectionState: ConnectionState;
   speaking?: boolean;
   audioMuted?: boolean;
   videoMuted?: boolean;
@@ -48,7 +49,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
   (
     {
       name,
-      hasFeed,
+      connectionState,
       speaking,
       audioMuted,
       videoMuted,
@@ -72,7 +73,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
     const { t } = useTranslation();
 
     const toolbarButtons: JSX.Element[] = [];
-    if (hasFeed && !isLocal) {
+    if (connectionState == ConnectionState.Connected && !isLocal) {
       toolbarButtons.push(
         <AudioButton
           key="localVolume"
@@ -94,7 +95,19 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }
 
-    const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name });
+    let caption: string;
+    switch (connectionState) {
+      case ConnectionState.EstablishingCall:
+        caption = t("{{name}} (Connecting...)", { name });
+        break;
+      case ConnectionState.WaitMedia:
+        // not strictly true, but probably easier to understand than, "Waiting for media"
+        caption = t("{{name}} (Waiting for video...)", { name });
+        break;
+      case ConnectionState.Connected:
+        caption = name;
+        break;
+    }
 
     return (
       <animated.div

src/video-grid/VideoTileContainer.tsx

@@ -98,7 +98,7 @@ export function VideoTileContainer({
         videoMuted={videoMuted}
         screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
         name={rawDisplayName}
-        hasFeed={Boolean(item.callFeed)}
+        connectionState={item.connectionState}
         ref={tileRef}
         mediaRef={mediaRef}
         avatar={getAvatar && getAvatar(item.member, width, height)}

yarn.lock

@@ -10196,9 +10196,9 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c6ee258789c9e01d328b5d9158b5b372e3a0da82":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3f1c3392d45b0fc054c3788cc6c043cd5b4fb730":
   version "21.1.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c6ee258789c9e01d328b5d9158b5b372e3a0da82"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3f1c3392d45b0fc054c3788cc6c043cd5b4fb730"
   dependencies:
     "@babel/runtime" "^7.12.5"
     "@types/sdp-transform" "^2.4.5"