Merge remote-tracking branch 'upstream/feature_sfu' into feature_simulcast

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

src/ClientContext.tsx

@@ -30,7 +30,11 @@ import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
 
 import { ErrorView } from "./FullScreenView";
-import { initClient, CryptoStoreIntegrityError } from "./matrix-utils";
+import {
+  initClient,
+  CryptoStoreIntegrityError,
+  fallbackICEServerAllowed,
+} from "./matrix-utils";
 import { widget } from "./widget";
 import { PosthogAnalytics, RegistrationType } from "./PosthogAnalytics";
 import { translatedError } from "./TranslatedError";
@@ -139,6 +143,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
                   accessToken: access_token,
                   userId: user_id,
                   deviceId: device_id,
+                  fallbackICEServerAllowed: fallbackICEServerAllowed,
                 },
                 true
               ),
@@ -154,6 +159,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
                     accessToken: access_token,
                     userId: user_id,
                     deviceId: device_id,
+                    fallbackICEServerAllowed: fallbackICEServerAllowed,
                   },
                   false // Don't need the crypto store just to log out
                 );

src/matrix-utils.ts

@@ -99,12 +99,17 @@ export async function initClient(
     localSfuDeviceId: Config.get().temp_sfu?.device_id,
   } as ICreateClientOpts;
 
-  if (indexedDB && localStorage && !import.meta.env.DEV) {
+  if (indexedDB && localStorage) {
     baseOpts.store = new IndexedDBStore({
       indexedDB: window.indexedDB,
       localStorage,
       dbName: SYNC_STORE_NAME,
-      workerFactory: () => new IndexedDBWorker(),
+      // We can't use the worker in dev mode because Vite simply doesn't bundle workers
+      // in dev mode: it expects them to use native modules. Ours don't, and even then only
+      // Chrome supports it. (It bundles them fine in production mode.)
+      workerFactory: import.meta.env.DEV
+        ? undefined
+        : () => new IndexedDBWorker(),
     });
   } else if (localStorage) {
     baseOpts.store = new MemoryStore({ localStorage });

src/room/GroupCallView.tsx

@@ -75,6 +75,7 @@ export function GroupCallView({
     toggleLocalVideoMuted,
     toggleMicrophoneMuted,
     toggleScreensharing,
+    setMicrophoneMuted,
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
@@ -251,6 +252,7 @@ export function GroupCallView({
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
           toggleMicrophoneMuted={toggleMicrophoneMuted}
+          setMicrophoneMuted={setMicrophoneMuted}
           userMediaFeeds={userMediaFeeds}
           activeSpeaker={activeSpeaker}
           onLeave={onLeave}

src/room/InCallView.tsx

@@ -63,6 +63,7 @@ import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
 import { ParticipantInfo } from "./useGroupCall";
 import { TileDescriptor } from "../video-grid/TileDescriptor";
 import { AudioSink } from "../video-grid/AudioSink";
+import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -81,6 +82,7 @@ interface Props {
   toggleLocalVideoMuted: () => void;
   toggleMicrophoneMuted: () => void;
   toggleScreensharing: () => void;
+  setMicrophoneMuted: (muted: boolean) => void;
   userMediaFeeds: CallFeed[];
   activeSpeaker: CallFeed | null;
   onLeave: () => void;
@@ -101,6 +103,7 @@ export function InCallView({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
+  setMicrophoneMuted,
   userMediaFeeds,
   activeSpeaker,
   onLeave,
@@ -141,6 +144,13 @@ export function InCallView({
 
   const { hideScreensharing } = useUrlParams();
 
+  useCallViewKeyboardShortcuts(
+    !feedbackModalState.isOpen,
+    toggleMicrophoneMuted,
+    toggleLocalVideoMuted,
+    setMicrophoneMuted
+  );
+
   useEffect(() => {
     widget?.api.transport.send(
       layout === "freedom"

src/room/useGroupCall.ts

@@ -32,8 +32,6 @@ import { usePageUnload } from "./usePageUnload";
 import { PosthogAnalytics } from "../PosthogAnalytics";
 import { TranslatedError, translatedError } from "../TranslatedError";
 import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget";
-import { getSetting } from "../settings/useSetting";
-import { useEventTarget } from "../useEvents";
 
 export enum ConnectionState {
   EstablishingCall = "establishing call", // call hasn't been established yet
@@ -60,6 +58,7 @@ export interface UseGroupCallReturnType {
   toggleLocalVideoMuted: () => void;
   toggleMicrophoneMuted: () => void;
   toggleScreensharing: () => void;
+  setMicrophoneMuted: (muted: boolean) => void;
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
@@ -472,68 +471,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     }
   }, [t, updateState]);
 
-  const [spacebarHeld, setSpacebarHeld] = useState(false);
-
-  useEventTarget(
-    window,
-    "keydown",
-    useCallback(
-      (event: KeyboardEvent) => {
-        // Check if keyboard shortcuts are enabled
-        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
-        if (!keyboardShortcuts) {
-          return;
-        }
-
-        if (event.key === "m") {
-          toggleMicrophoneMuted();
-        } else if (event.key == "v") {
-          toggleLocalVideoMuted();
-        } else if (event.key === " ") {
-          setSpacebarHeld(true);
-          setMicrophoneMuted(false);
-        }
-      },
-      [
-        toggleLocalVideoMuted,
-        toggleMicrophoneMuted,
-        setMicrophoneMuted,
-        setSpacebarHeld,
-      ]
-    )
-  );
-
-  useEventTarget(
-    window,
-    "keyup",
-    useCallback(
-      (event: KeyboardEvent) => {
-        // Check if keyboard shortcuts are enabled
-        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
-        if (!keyboardShortcuts) {
-          return;
-        }
-
-        if (event.key === " ") {
-          setSpacebarHeld(false);
-          setMicrophoneMuted(true);
-        }
-      },
-      [setMicrophoneMuted, setSpacebarHeld]
-    )
-  );
-
-  useEventTarget(
-    window,
-    "blur",
-    useCallback(() => {
-      if (spacebarHeld) {
-        setSpacebarHeld(false);
-        setMicrophoneMuted(true);
-      }
-    }, [setMicrophoneMuted, setSpacebarHeld, spacebarHeld])
-  );
-
   return {
     state,
     localCallFeed,
@@ -548,6 +485,7 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     toggleLocalVideoMuted,
     toggleMicrophoneMuted,
     toggleScreensharing,
+    setMicrophoneMuted,
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,

src/useCallViewKeyboardShortcuts.ts

@@ -0,0 +1,93 @@
+/*
+Copyright 2022-2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { useCallback, useState } from "react";
+
+import { getSetting } from "./settings/useSetting";
+import { useEventTarget } from "./useEvents";
+
+export function useCallViewKeyboardShortcuts(
+  enabled: boolean,
+  toggleMicrophoneMuted: () => void,
+  toggleLocalVideoMuted: () => void,
+  setMicrophoneMuted: (muted: boolean) => void
+) {
+  const [spacebarHeld, setSpacebarHeld] = useState(false);
+
+  useEventTarget(
+    window,
+    "keydown",
+    useCallback(
+      (event: KeyboardEvent) => {
+        if (!enabled) return;
+        // Check if keyboard shortcuts are enabled
+        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
+        if (!keyboardShortcuts) {
+          return;
+        }
+
+        if (event.key === "m") {
+          toggleMicrophoneMuted();
+        } else if (event.key == "v") {
+          toggleLocalVideoMuted();
+        } else if (event.key === " " && !spacebarHeld) {
+          setSpacebarHeld(true);
+          setMicrophoneMuted(false);
+        }
+      },
+      [
+        enabled,
+        spacebarHeld,
+        toggleLocalVideoMuted,
+        toggleMicrophoneMuted,
+        setMicrophoneMuted,
+        setSpacebarHeld,
+      ]
+    )
+  );
+
+  useEventTarget(
+    window,
+    "keyup",
+    useCallback(
+      (event: KeyboardEvent) => {
+        if (!enabled) return;
+        // Check if keyboard shortcuts are enabled
+        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
+        if (!keyboardShortcuts) {
+          return;
+        }
+
+        if (event.key === " ") {
+          setSpacebarHeld(false);
+          setMicrophoneMuted(true);
+        }
+      },
+      [enabled, setMicrophoneMuted, setSpacebarHeld]
+    )
+  );
+
+  useEventTarget(
+    window,
+    "blur",
+    useCallback(() => {
+      if (spacebarHeld) {
+        setSpacebarHeld(false);
+        setMicrophoneMuted(true);
+      }
+    }, [setMicrophoneMuted, setSpacebarHeld, spacebarHeld])
+  );
+}

src/video-grid/VideoTile.tsx

@@ -36,6 +36,7 @@ interface Props {
   mediaRef?: React.RefObject<MediaElement>;
   onOptionsPress?: () => void;
   localVolume?: number;
+  hasAudio?: boolean;
   maximised?: boolean;
   fullscreen?: boolean;
   onFullscreen?: () => void;
@@ -58,6 +59,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       mediaRef,
       onOptionsPress,
       localVolume,
+      hasAudio,
       maximised,
       fullscreen,
       onFullscreen,
@@ -74,14 +76,16 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
 
     const toolbarButtons: JSX.Element[] = [];
     if (connectionState == ConnectionState.Connected && !isLocal) {
-      toolbarButtons.push(
-        <AudioButton
-          key="localVolume"
-          className={styles.button}
-          volume={localVolume}
-          onPress={onOptionsPress}
-        />
-      );
+      if (hasAudio) {
+        toolbarButtons.push(
+          <AudioButton
+            key="localVolume"
+            className={styles.button}
+            volume={localVolume}
+            onPress={onOptionsPress}
+          />
+        );
+      }
 
       if (screenshare) {
         toolbarButtons.push(
@@ -137,7 +141,13 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             </div>
           ) : (
             <div className={classNames(styles.infoBubble, styles.memberName)}>
-              {audioMuted && !videoMuted && <MicMutedIcon />}
+              {
+                /* If the user is speaking, it's safe to say they're unmuted.
+                Mute state is currently sent over to-device messages, which
+                aren't quite real-time, so this is an important kludge to make
+                sure no one appears muted when they've clearly begun talking. */
+                audioMuted && !videoMuted && !speaking && <MicMutedIcon />
+              }
               {videoMuted && <VideoMutedIcon />}
               <span title={caption}>{caption}</span>
             </div>

src/video-grid/VideoTileContainer.tsx

@@ -62,6 +62,7 @@ export function VideoTileContainer({
     audioMuted,
     videoMuted,
     localVolume,
+    hasAudio,
     speaking,
     stream,
     purpose,
@@ -117,6 +118,7 @@ export function VideoTileContainer({
         avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
+        hasAudio={hasAudio}
         maximised={maximised}
         fullscreen={fullscreen}
         onFullscreen={onFullscreenCallback}

src/video-grid/useCallFeed.ts

@@ -25,6 +25,7 @@ interface CallFeedState {
   videoMuted: boolean;
   audioMuted: boolean;
   localVolume: number;
+  hasAudio: boolean;
   disposed: boolean | undefined;
   stream: MediaStream | undefined;
   purpose: SDPStreamMetadataPurpose | undefined;
@@ -38,6 +39,7 @@ function getCallFeedState(callFeed: CallFeed | undefined): CallFeedState {
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,
     audioMuted: callFeed ? callFeed.isAudioMuted() : true,
     localVolume: callFeed ? callFeed.getLocalVolume() : 0,
+    hasAudio: callFeed ? callFeed.stream.getAudioTracks().length >= 1 : false,
     disposed: callFeed ? callFeed.disposed : undefined,
     stream: callFeed ? callFeed.stream : undefined,
     purpose: callFeed ? callFeed.purpose : undefined,