diff --git a/package.json b/package.json index 6227b91..de7eec7 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "i18next": "^21.10.0", "i18next-browser-languagedetector": "^6.1.8", "i18next-http-backend": "^1.4.4", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c6ee258789c9e01d328b5d9158b5b372e3a0da82", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3f1c3392d45b0fc054c3788cc6c043cd5b4fb730", "matrix-widget-api": "^1.0.0", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index b05182a..7da620b 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -79,6 +79,7 @@ export function GroupCallView({ isScreensharing, screenshareFeeds, participants, + calls, unencryptedEventsFromUsers, } = useGroupCall(groupCall); @@ -235,6 +236,7 @@ export function GroupCallView({ roomName={groupCall.room.name} avatarUrl={avatarUrl} participants={participants} + calls={calls} microphoneMuted={microphoneMuted} localVideoMuted={localVideoMuted} toggleLocalVideoMuted={toggleLocalVideoMuted} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b4de4b9..d3d0fc0 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -14,7 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { useEffect, useCallback, useMemo, useRef } from "react"; +import React, { + useEffect, + useCallback, + useMemo, + useRef, + useState, +} from "react"; import { usePreventScroll } from "@react-aria/overlays"; import useMeasure from "react-use-measure"; import { ResizeObserver } from "@juggle/resize-observer"; @@ -25,6 +31,11 @@ import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; +import { + CallEvent, + CallState, + MatrixCall, +} from "matrix-js-sdk/src/webrtc/call"; import type { IWidgetApiRequest } from "matrix-widget-api"; import styles from "./InCallView.module.css"; @@ -73,6 +84,7 @@ interface Props { client: MatrixClient; groupCall: GroupCall; participants: RoomMember[]; + calls: MatrixCall[]; roomName: string; avatarUrl: string; microphoneMuted: boolean; @@ -90,6 +102,12 @@ interface Props { hideHeader: boolean; } +export enum ConnectionState { + ESTABLISHING_CALL = "establishing call", // call hasn't been established yet + WAIT_MEDIA = "wait_media", // call is set up, waiting for ICE to connect + CONNECTED = "connected", // media is flowing +} + // Represents something that should get a tile on the layout, // ie. a user's video feed or a screen share feed. export interface TileDescriptor { @@ -99,12 +117,14 @@ export interface TileDescriptor { presenter: boolean; callFeed?: CallFeed; isLocal?: boolean; + connectionState: ConnectionState; } export function InCallView({ client, groupCall, participants, + calls, roomName, avatarUrl, microphoneMuted, @@ -154,6 +174,46 @@ export function InCallView({ const { hideScreensharing } = useUrlParams(); + const makeConnectionStatesMap = useCallback(() => { + const newConnStates = new Map(); + for (const participant of participants) { + const userCall = groupCall.getCallByUserId(participant.userId); + const feed = userMediaFeeds.find((f) => f.userId === participant.userId); + let connectionState = ConnectionState.ESTABLISHING_CALL; + if (feed && feed.isLocal()) { + connectionState = ConnectionState.CONNECTED; + } else if (userCall) { + if (userCall.state === CallState.Connected) { + connectionState = ConnectionState.CONNECTED; + } else if (userCall.state === CallState.Connecting) { + connectionState = ConnectionState.WAIT_MEDIA; + } + } + newConnStates.set(participant.userId, connectionState); + } + return newConnStates; + }, [groupCall, participants, userMediaFeeds]); + + const [connStates, setConnStates] = useState( + new Map() + ); + + const updateConnectionStates = useCallback(() => { + setConnStates(makeConnectionStatesMap()); + }, [setConnStates, makeConnectionStatesMap]); + + useEffect(() => { + for (const call of calls) { + call.on(CallEvent.State, updateConnectionStates); + } + + return () => { + for (const call of calls) { + call.off(CallEvent.State, updateConnectionStates); + } + }; + }, [calls, updateConnectionStates]); + useEffect(() => { widget?.api.transport.send( layout === "freedom" @@ -208,6 +268,7 @@ export function InCallView({ focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker, isLocal: p.userId === client.getUserId(), presenter: false, + connectionState: connStates.get(p.userId), }); } @@ -231,11 +292,19 @@ export function InCallView({ focused: true, isLocal: screenshareFeed.isLocal(), presenter: false, + connectionState: ConnectionState.CONNECTED, // by definition since the screen shares arrived on the same connection }); } return tileDescriptors; - }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]); + }, [ + client, + participants, + userMediaFeeds, + activeSpeaker, + screenshareFeeds, + connStates, + ]); // The maximised participant: either the participant that the user has // manually put in fullscreen, or the focused (active) participant if the diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx index b8f3676..54f0cdf 100644 --- a/src/video-grid/VideoGrid.stories.tsx +++ b/src/video-grid/VideoGrid.stories.tsx @@ -21,7 +21,7 @@ import { RoomMember } from "matrix-js-sdk"; import { VideoGrid, useVideoGridLayout } from "./VideoGrid"; import { VideoTile } from "./VideoTile"; import { Button } from "../button"; -import { TileDescriptor } from "../room/InCallView"; +import { ConnectionState, TileDescriptor } from "../room/InCallView"; export default { title: "VideoGrid", @@ -41,6 +41,7 @@ export const ParticipantsTest = () => { member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`), focused: false, presenter: false, + connectionState: ConnectionState.CONNECTED, })), [participantCount] ); @@ -79,7 +80,7 @@ export const ParticipantsTest = () => { key={item.id} name={`User ${item.id}`} disableSpeakingIndicator={items.length < 3} - hasFeed={true} + connectionState={ConnectionState.CONNECTED} {...rest} /> )} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 2cef8e2..693475b 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -23,10 +23,11 @@ import styles from "./VideoTile.module.css"; import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg"; import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg"; import { AudioButton, FullscreenButton } from "../button/Button"; +import { ConnectionState } from "../room/InCallView"; interface Props { name: string; - hasFeed: Boolean; + connectionState: ConnectionState; speaking?: boolean; audioMuted?: boolean; videoMuted?: boolean; @@ -48,7 +49,7 @@ export const VideoTile = forwardRef( ( { name, - hasFeed, + connectionState, speaking, audioMuted, videoMuted, @@ -72,7 +73,7 @@ export const VideoTile = forwardRef( const { t } = useTranslation(); const toolbarButtons: JSX.Element[] = []; - if (hasFeed && !isLocal) { + if (connectionState == ConnectionState.CONNECTED && !isLocal) { toolbarButtons.push( ( } } - const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name }); + let caption: string; + switch (connectionState) { + case ConnectionState.ESTABLISHING_CALL: + caption = t("{{name}} (Connecting...)", { name }); + + break; + case ConnectionState.WAIT_MEDIA: + // not strictly true, but probably easier to understand than, "Waiting for media" + caption = t("{{name}} (Waiting for video...)", { name }); + break; + case ConnectionState.CONNECTED: + caption = name; + break; + } return (