Add widget actions for joining and leaving calls and switching layouts

These actions are processed lazily to ensure that even if the app takes a while to start up, they won't be missed.

src/room/GroupCallView.tsx

@@ -19,6 +19,8 @@ import { useHistory } from "react-router-dom";
 import { GroupCall, GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 
+import type { IWidgetApiRequest } from "matrix-widget-api";
+import { widget, ElementWidgetActions, JoinCallData } from "../widget";
 import { useGroupCall } from "./useGroupCall";
 import { ErrorView, FullScreenView } from "../FullScreenView";
 import { LobbyView } from "./LobbyView";
@@ -28,6 +30,8 @@ import { CallEndedView } from "./CallEndedView";
 import { useRoomAvatar } from "./useRoomAvatar";
 import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler";
 import { useLocationNavigation } from "../useLocationNavigation";
+import { useMediaHandler } from "../settings/useMediaHandler";
+
 declare global {
   interface Window {
     groupCall?: GroupCall;
@@ -38,6 +42,7 @@ interface Props {
   client: MatrixClient;
   isPasswordlessUser: boolean;
   isEmbedded: boolean;
+  preload: boolean;
   hideHeader: boolean;
   roomIdOrAlias: string;
   groupCall: GroupCall;
@@ -47,6 +52,7 @@ export function GroupCallView({
   client,
   isPasswordlessUser,
   isEmbedded,
+  preload,
   hideHeader,
   roomIdOrAlias,
   groupCall,
@@ -73,14 +79,50 @@ export function GroupCallView({
     unencryptedEventsFromUsers,
   } = useGroupCall(groupCall);
 
+  const { setAudioInput, setVideoInput } = useMediaHandler();
+
   const avatarUrl = useRoomAvatar(groupCall.room);
 
   useEffect(() => {
     window.groupCall = groupCall;
+    return () => {
+      delete window.groupCall;
+    };
+  }, [groupCall]);
 
-    // In embedded mode, bypass the lobby and just enter the call straight away
+  useEffect(() => {
-    if (isEmbedded) groupCall.enter();
+    if (widget && preload) {
-  }, [groupCall, isEmbedded]);
+      // In preload mode, wait for a join action before entering
+      const onJoin = async (ev: CustomEvent<IWidgetApiRequest>) => {
+        const { audioInput, videoInput } = ev.detail
+          .data as unknown as JoinCallData;
+        if (audioInput !== null) setAudioInput(audioInput);
+        if (videoInput !== null) setVideoInput(videoInput);
+        await Promise.all([
+          groupCall.setMicrophoneMuted(audioInput === null),
+          groupCall.setLocalVideoMuted(videoInput === null),
+        ]);
+
+        await groupCall.enter();
+        await Promise.all([
+          widget.api.setAlwaysOnScreen(true),
+          widget.api.transport.reply(ev.detail, {}),
+        ]);
+      };
+
+      widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
+      return () => {
+        widget.lazyActions.off(ElementWidgetActions.JoinCall, onJoin);
+      };
+    }
+  }, [groupCall, preload, setAudioInput, setVideoInput]);
+
+  useEffect(() => {
+    if (isEmbedded && !preload) {
+      // In embedded mode, bypass the lobby and just enter the call straight away
+      groupCall.enter();
+    }
+  }, [groupCall, isEmbedded, preload]);
 
   useSentryGroupCallHandler(groupCall);
 
@@ -90,13 +132,31 @@ export function GroupCallView({
   const history = useHistory();
 
   const onLeave = useCallback(() => {
-    setLeft(true);
     leave();
+    if (widget) {
+      widget.api.transport.send(ElementWidgetActions.HangupCall, {});
+      widget.api.setAlwaysOnScreen(false);
+    }
 
-    if (!isPasswordlessUser) {
+    if (isPasswordlessUser) {
+      setLeft(true);
+    } else if (!isEmbedded) {
       history.push("/");
     }
-  }, [leave, isPasswordlessUser, history]);
+  }, [leave, isPasswordlessUser, isEmbedded, history]);
+
+  useEffect(() => {
+    if (widget && state === GroupCallState.Entered) {
+      const onHangup = async (ev: CustomEvent<IWidgetApiRequest>) => {
+        leave();
+        await widget.api.transport.reply(ev.detail, {});
+      };
+      widget.lazyActions.once(ElementWidgetActions.HangupCall, onHangup);
+      return () => {
+        widget.lazyActions.off(ElementWidgetActions.HangupCall, onHangup);
+      };
+    }
+  }, [groupCall, state, leave]);
 
   if (error) {
     return <ErrorView error={error} />;
@@ -148,33 +208,33 @@ export function GroupCallView({
     );
   } else if (left) {
     return <CallEndedView client={client} />;
+  } else if (preload) {
+    return null;
+  } else if (isEmbedded) {
+    return (
+      <FullScreenView>
+        <h1>Loading room...</h1>
+      </FullScreenView>
+    );
   } else {
-    if (isEmbedded) {
+    return (
-      return (
+      <LobbyView
-        <FullScreenView>
+        client={client}
-          <h1>Loading room...</h1>
+        groupCall={groupCall}
-        </FullScreenView>
+        roomName={groupCall.room.name}
-      );
+        avatarUrl={avatarUrl}
-    } else {
+        state={state}
-      return (
+        onInitLocalCallFeed={initLocalCallFeed}
-        <LobbyView
+        localCallFeed={localCallFeed}
-          client={client}
+        onEnter={enter}
-          groupCall={groupCall}
+        microphoneMuted={microphoneMuted}
-          roomName={groupCall.room.name}
+        localVideoMuted={localVideoMuted}
-          avatarUrl={avatarUrl}
+        toggleLocalVideoMuted={toggleLocalVideoMuted}
-          state={state}
+        toggleMicrophoneMuted={toggleMicrophoneMuted}
-          onInitLocalCallFeed={initLocalCallFeed}
+        roomIdOrAlias={roomIdOrAlias}
-          localCallFeed={localCallFeed}
+        isEmbedded={isEmbedded}
-          onEnter={enter}
+        hideHeader={hideHeader}
-          microphoneMuted={microphoneMuted}
+      />
-          localVideoMuted={localVideoMuted}
+    );
-          toggleLocalVideoMuted={toggleLocalVideoMuted}
-          toggleMicrophoneMuted={toggleMicrophoneMuted}
-          roomIdOrAlias={roomIdOrAlias}
-          isEmbedded={isEmbedded}
-          hideHeader={hideHeader}
-        />
-      );
-    }
   }
 }

src/room/InCallView.tsx

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useCallback, useMemo, useRef } from "react";
+import React, { useEffect, useCallback, useMemo, useRef } from "react";
 import { usePreventScroll } from "@react-aria/overlays";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
@@ -22,6 +22,7 @@ import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
 import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
 import classNames from "classnames";
 
+import type { IWidgetApiRequest } from "matrix-widget-api";
 import styles from "./InCallView.module.css";
 import {
   HangupButton,
@@ -52,6 +53,7 @@ import { useAudioContext } from "../video-grid/useMediaStream";
 import { useFullscreen } from "../video-grid/useFullscreen";
 import { AudioContainer } from "../video-grid/AudioContainer";
 import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice";
+import { widget, ElementWidgetActions } from "../widget";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -124,6 +126,42 @@ export function InCallView({
 
   useAudioOutputDevice(audioRef, audioOutput);
 
+  useEffect(() => {
+    widget?.api.transport.send(
+      layout === "freedom"
+        ? ElementWidgetActions.TileLayout
+        : ElementWidgetActions.SpotlightLayout,
+      {}
+    );
+  }, [layout]);
+
+  useEffect(() => {
+    if (widget) {
+      const onTileLayout = async (ev: CustomEvent<IWidgetApiRequest>) => {
+        setLayout("freedom");
+        await widget.api.transport.reply(ev.detail, {});
+      };
+      const onSpotlightLayout = async (ev: CustomEvent<IWidgetApiRequest>) => {
+        setLayout("spotlight");
+        await widget.api.transport.reply(ev.detail, {});
+      };
+
+      widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout);
+      widget.lazyActions.on(
+        ElementWidgetActions.SpotlightLayout,
+        onSpotlightLayout
+      );
+
+      return () => {
+        widget.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout);
+        widget.lazyActions.off(
+          ElementWidgetActions.SpotlightLayout,
+          onSpotlightLayout
+        );
+      };
+    }
+  }, [setLayout]);
+
   const items = useMemo(() => {
     const participants: Participant[] = [];
 

src/room/RoomPage.tsx

@@ -14,8 +14,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { FC, useEffect, useState } from "react";
+import React, { FC, useEffect, useState, useCallback } from "react";
 
+import type { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
 import { useClient } from "../ClientContext";
 import { ErrorView, LoadingView } from "../FullScreenView";
 import { RoomAuthView } from "./RoomAuthView";
@@ -34,6 +35,7 @@ export const RoomPage: FC = () => {
     roomId,
     viaServers,
     isEmbedded,
+    preload,
     hideHeader,
     isPtt,
     displayName,
@@ -68,10 +70,11 @@ export const RoomPage: FC = () => {
         groupCall={groupCall}
         isPasswordlessUser={isPasswordlessUser}
         isEmbedded={isEmbedded}
+        preload={preload}
         hideHeader={hideHeader}
       />
     ),
-    [client, roomIdOrAlias, isPasswordlessUser, isEmbedded, hideHeader]
+    [client, roomIdOrAlias, isPasswordlessUser, isEmbedded, preload, hideHeader]
   );
 
   if (loading || isRegistering) {

src/room/useRoomParams.ts

@@ -24,6 +24,9 @@ export interface RoomParams {
   // Whether the app is running in embedded mode, and should keep the user
   // confined to the current room
   isEmbedded: boolean;
+  // Whether the app should pause before joining the call until it sees an
+  // io.element.join widget action, allowing it to be preloaded
+  preload: boolean;
   // Whether to hide the room header when in a call
   hideHeader: boolean;
   // Whether to start a walkie-talkie call instead of a video call
@@ -77,6 +80,7 @@ export const getRoomParams = (
     roomId: getParam("roomId"),
     viaServers: getAllParams("via"),
     isEmbedded: hasParam("embed"),
+    preload: hasParam("preload"),
     hideHeader: hasParam("hideHeader"),
     isPtt: hasParam("ptt"),
     e2eEnabled: getParam("enableE2e") !== "false", // Defaults to true