Merge remote-tracking branch 'upstream/feature_sfu' into feature_simulcast

Signed-off-by: Šimon Brandner <simon.bra.ag@gmail.com>

config/nginx.conf

@@ -2,9 +2,24 @@ server {
     listen       8080;
     server_name  localhost;
 
+    root   /app;
+
     location / {
-        root   /app;
+        # disable cache entriely by default (apart from Etag which is accurate enough)
+        add_header Cache-Control 'private no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0';
+        if_modified_since off;
+        expires off;
+        # also turn off last-modified since they are just the timestamps of the file in the docker image
+        # and may or may not bear any resemblance to when the resource changed
+        add_header Last-Modified "";
+
         try_files $uri /$uri /index.html;
     }
+
+    # assets can be cached because they have hashed filenames
+    location /assets {
+        expires 1w;
+        add_header Cache-Control "public, no-transform";
+    }
 }
 

package.json

@@ -45,7 +45,7 @@
     "i18next": "^21.10.0",
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#77ae9947984fb9f1c1bee9f28e6dfda75539e739",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e1a14812fd9f0a261b0e823e3cc68d9ee93b8f11",
     "matrix-widget-api": "^1.0.0",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",

public/locales/de/app.json

@@ -137,5 +137,7 @@
   "Whether to enable single-key keyboard shortcuts, e.g. 'm' to mute/unmute the mic.": "Ob Tastenkürzel mit nur einer Taste aktiviert sein sollen, z. B. „m“ um das Mikrofon stumm/aktiv zu schalten.",
   "Single-key keyboard shortcuts": "Ein-Tasten-Tastenkürzel",
   "{{name}} (Waiting for video...)": "{{name}} (Warte auf Video …)",
-  "This feature is only supported on Firefox.": "Diese Funktion wird nur in Firefox unterstützt."
+  "This feature is only supported on Firefox.": "Diese Funktion wird nur in Firefox unterstützt.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Übermittelte Problemberichte helfen uns, Fehler zu beheben.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Hoppla, etwas ist schiefgelaufen.</0>"
 }

public/locales/et/app.json

@@ -137,5 +137,7 @@
   "Whether to enable single-key keyboard shortcuts, e.g. 'm' to mute/unmute the mic.": "Kas kasutame üheklahvilisi kiirklahve, näiteks „m“ mikrofoni sisse/välja lülitamiseks.",
   "Single-key keyboard shortcuts": "Üheklahvilised kiirklahvid",
   "{{name}} (Waiting for video...)": "{{name}} (Ootame videovoo algust...)",
-  "This feature is only supported on Firefox.": "See funktsionaalsus on toetatud vaid Firefoxis."
+  "This feature is only supported on Firefox.": "See funktsionaalsus on toetatud vaid Firefoxis.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Kui saadad meile vealogid, siis on lihtsam vea põhjust otsida.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Ohoo, midagi on nüüd katki.</0>"
 }

public/locales/fr/app.json

@@ -137,5 +137,7 @@
   "Whether to enable single-key keyboard shortcuts, e.g. 'm' to mute/unmute the mic.": "Bascule sur les raccourcis clavier à touche unique, par exemple « m » pour désactiver / activer le micro.",
   "Single-key keyboard shortcuts": "Raccourcis clavier en une touche",
   "{{name}} (Waiting for video...)": "{{name}} (En attente de vidéo…)",
-  "This feature is only supported on Firefox.": "Cette fonctionnalité est prise en charge dans Firefox uniquement."
+  "This feature is only supported on Firefox.": "Cette fonctionnalité est prise en charge dans Firefox uniquement.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Soumettre les journaux de débogage nous aidera à déterminer le problème.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Oups, quelque chose s’est mal passé.</0>"
 }

public/locales/id/app.json

@@ -137,5 +137,7 @@
   "Whether to enable single-key keyboard shortcuts, e.g. 'm' to mute/unmute the mic.": "Apakah pintasan papan ketik seharusnya diaktifkan, mis. 'm' untuk membisukan/menyuarakan mikrofon.",
   "Single-key keyboard shortcuts": "Pintasan papan ketik satu tombol",
   "{{name}} (Waiting for video...)": "{{name}} (Menunggu video...)",
-  "This feature is only supported on Firefox.": "Fitur ini hanya didukung di Firefox."
+  "This feature is only supported on Firefox.": "Fitur ini hanya didukung di Firefox.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Mengirim catatan pengawakutuan akan membantu kami melacak masalahnya.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Aduh, ada yang salah.</0>"
 }

public/locales/sk/app.json

@@ -137,5 +137,7 @@
   "{{displayName}}, your call is now ended": "{{displayName}}, váš hovor je teraz ukončený",
   "{{count}} people connected|other": "{{count}} osôb pripojených",
   "{{count}} people connected|one": "{{count}} osoba pripojená",
-  "This feature is only supported on Firefox.": "Táto funkcia je podporovaná len v prehliadači Firefox."
+  "This feature is only supported on Firefox.": "Táto funkcia je podporovaná len v prehliadači Firefox.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Odoslanie záznamov ladenia nám pomôže nájsť problém.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Hups, niečo sa pokazilo.</0>"
 }

public/locales/uk/app.json

@@ -137,5 +137,7 @@
   "Whether to enable single-key keyboard shortcuts, e.g. 'm' to mute/unmute the mic.": "Чи вмикати/вимикати мікрофон однією клавішею, наприклад, «m» для ввімкнення/вимкнення мікрофона.",
   "Single-key keyboard shortcuts": "Одноклавішні комбінації клавіш",
   "{{name}} (Waiting for video...)": "{{name}} (Очікування на відео...)",
-  "This feature is only supported on Firefox.": "Ця функція підтримується лише в браузері Firefox."
+  "This feature is only supported on Firefox.": "Ця функція підтримується лише в браузері Firefox.",
+  "<0>Submitting debug logs will help us track down the problem.</0>": "<0>Надсилання журналів зневадження допоможе нам виявити проблему.</0>",
+  "<0>Oops, something's gone wrong.</0>": "<0>Йой, щось пішло не за планом.</0>"
 }

src/ClientContext.tsx

@@ -30,7 +30,11 @@ import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
 
 import { ErrorView } from "./FullScreenView";
-import { initClient, CryptoStoreIntegrityError } from "./matrix-utils";
+import {
+  initClient,
+  CryptoStoreIntegrityError,
+  fallbackICEServerAllowed,
+} from "./matrix-utils";
 import { widget } from "./widget";
 import { PosthogAnalytics, RegistrationType } from "./PosthogAnalytics";
 import { translatedError } from "./TranslatedError";
@@ -139,6 +143,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
                   accessToken: access_token,
                   userId: user_id,
                   deviceId: device_id,
+                  fallbackICEServerAllowed: fallbackICEServerAllowed,
                 },
                 true
               ),
@@ -154,6 +159,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
                     accessToken: access_token,
                     userId: user_id,
                     deviceId: device_id,
+                    fallbackICEServerAllowed: fallbackICEServerAllowed,
                   },
                   false // Don't need the crypto store just to log out
                 );

src/matrix-utils.ts

@@ -99,12 +99,17 @@ export async function initClient(
     localSfuDeviceId: Config.get().temp_sfu?.device_id,
   } as ICreateClientOpts;
 
-  if (indexedDB && localStorage && !import.meta.env.DEV) {
+  if (indexedDB && localStorage) {
     baseOpts.store = new IndexedDBStore({
       indexedDB: window.indexedDB,
       localStorage,
       dbName: SYNC_STORE_NAME,
-      workerFactory: () => new IndexedDBWorker(),
+      // We can't use the worker in dev mode because Vite simply doesn't bundle workers
+      // in dev mode: it expects them to use native modules. Ours don't, and even then only
+      // Chrome supports it. (It bundles them fine in production mode.)
+      workerFactory: import.meta.env.DEV
+        ? undefined
+        : () => new IndexedDBWorker(),
     });
   } else if (localStorage) {
     baseOpts.store = new MemoryStore({ localStorage });

src/room/GroupCallView.tsx

@@ -75,6 +75,7 @@ export function GroupCallView({
     toggleLocalVideoMuted,
     toggleMicrophoneMuted,
     toggleScreensharing,
+    setMicrophoneMuted,
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,
@@ -251,6 +252,7 @@ export function GroupCallView({
           localVideoMuted={localVideoMuted}
           toggleLocalVideoMuted={toggleLocalVideoMuted}
           toggleMicrophoneMuted={toggleMicrophoneMuted}
+          setMicrophoneMuted={setMicrophoneMuted}
           userMediaFeeds={userMediaFeeds}
           activeSpeaker={activeSpeaker}
           onLeave={onLeave}

src/room/InCallView.tsx

@@ -63,6 +63,7 @@ import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
 import { ParticipantInfo } from "./useGroupCall";
 import { TileDescriptor } from "../video-grid/TileDescriptor";
 import { AudioSink } from "../video-grid/AudioSink";
+import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -81,6 +82,7 @@ interface Props {
   toggleLocalVideoMuted: () => void;
   toggleMicrophoneMuted: () => void;
   toggleScreensharing: () => void;
+  setMicrophoneMuted: (muted: boolean) => void;
   userMediaFeeds: CallFeed[];
   activeSpeaker: CallFeed | null;
   onLeave: () => void;
@@ -101,6 +103,7 @@ export function InCallView({
   localVideoMuted,
   toggleLocalVideoMuted,
   toggleMicrophoneMuted,
+  setMicrophoneMuted,
   userMediaFeeds,
   activeSpeaker,
   onLeave,
@@ -141,6 +144,13 @@ export function InCallView({
 
   const { hideScreensharing } = useUrlParams();
 
+  useCallViewKeyboardShortcuts(
+    !feedbackModalState.isOpen,
+    toggleMicrophoneMuted,
+    toggleLocalVideoMuted,
+    setMicrophoneMuted
+  );
+
   useEffect(() => {
     widget?.api.transport.send(
       layout === "freedom"

src/room/useGroupCall.ts

@@ -32,8 +32,6 @@ import { usePageUnload } from "./usePageUnload";
 import { PosthogAnalytics } from "../PosthogAnalytics";
 import { TranslatedError, translatedError } from "../TranslatedError";
 import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget";
-import { getSetting } from "../settings/useSetting";
-import { useEventTarget } from "../useEvents";
 
 export enum ConnectionState {
   EstablishingCall = "establishing call", // call hasn't been established yet
@@ -60,6 +58,7 @@ export interface UseGroupCallReturnType {
   toggleLocalVideoMuted: () => void;
   toggleMicrophoneMuted: () => void;
   toggleScreensharing: () => void;
+  setMicrophoneMuted: (muted: boolean) => void;
   requestingScreenshare: boolean;
   isScreensharing: boolean;
   screenshareFeeds: CallFeed[];
@@ -472,68 +471,6 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     }
   }, [t, updateState]);
 
-  const [spacebarHeld, setSpacebarHeld] = useState(false);
-
-  useEventTarget(
-    window,
-    "keydown",
-    useCallback(
-      (event: KeyboardEvent) => {
-        // Check if keyboard shortcuts are enabled
-        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
-        if (!keyboardShortcuts) {
-          return;
-        }
-
-        if (event.key === "m") {
-          toggleMicrophoneMuted();
-        } else if (event.key == "v") {
-          toggleLocalVideoMuted();
-        } else if (event.key === " ") {
-          setSpacebarHeld(true);
-          setMicrophoneMuted(false);
-        }
-      },
-      [
-        toggleLocalVideoMuted,
-        toggleMicrophoneMuted,
-        setMicrophoneMuted,
-        setSpacebarHeld,
-      ]
-    )
-  );
-
-  useEventTarget(
-    window,
-    "keyup",
-    useCallback(
-      (event: KeyboardEvent) => {
-        // Check if keyboard shortcuts are enabled
-        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
-        if (!keyboardShortcuts) {
-          return;
-        }
-
-        if (event.key === " ") {
-          setSpacebarHeld(false);
-          setMicrophoneMuted(true);
-        }
-      },
-      [setMicrophoneMuted, setSpacebarHeld]
-    )
-  );
-
-  useEventTarget(
-    window,
-    "blur",
-    useCallback(() => {
-      if (spacebarHeld) {
-        setSpacebarHeld(false);
-        setMicrophoneMuted(true);
-      }
-    }, [setMicrophoneMuted, setSpacebarHeld, spacebarHeld])
-  );
-
   return {
     state,
     localCallFeed,
@@ -548,6 +485,7 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     toggleLocalVideoMuted,
     toggleMicrophoneMuted,
     toggleScreensharing,
+    setMicrophoneMuted,
     requestingScreenshare,
     isScreensharing,
     screenshareFeeds,

src/useCallViewKeyboardShortcuts.ts

@@ -0,0 +1,93 @@
+/*
+Copyright 2022-2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { useCallback, useState } from "react";
+
+import { getSetting } from "./settings/useSetting";
+import { useEventTarget } from "./useEvents";
+
+export function useCallViewKeyboardShortcuts(
+  enabled: boolean,
+  toggleMicrophoneMuted: () => void,
+  toggleLocalVideoMuted: () => void,
+  setMicrophoneMuted: (muted: boolean) => void
+) {
+  const [spacebarHeld, setSpacebarHeld] = useState(false);
+
+  useEventTarget(
+    window,
+    "keydown",
+    useCallback(
+      (event: KeyboardEvent) => {
+        if (!enabled) return;
+        // Check if keyboard shortcuts are enabled
+        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
+        if (!keyboardShortcuts) {
+          return;
+        }
+
+        if (event.key === "m") {
+          toggleMicrophoneMuted();
+        } else if (event.key == "v") {
+          toggleLocalVideoMuted();
+        } else if (event.key === " " && !spacebarHeld) {
+          setSpacebarHeld(true);
+          setMicrophoneMuted(false);
+        }
+      },
+      [
+        enabled,
+        spacebarHeld,
+        toggleLocalVideoMuted,
+        toggleMicrophoneMuted,
+        setMicrophoneMuted,
+        setSpacebarHeld,
+      ]
+    )
+  );
+
+  useEventTarget(
+    window,
+    "keyup",
+    useCallback(
+      (event: KeyboardEvent) => {
+        if (!enabled) return;
+        // Check if keyboard shortcuts are enabled
+        const keyboardShortcuts = getSetting("keyboard-shortcuts", true);
+        if (!keyboardShortcuts) {
+          return;
+        }
+
+        if (event.key === " ") {
+          setSpacebarHeld(false);
+          setMicrophoneMuted(true);
+        }
+      },
+      [enabled, setMicrophoneMuted, setSpacebarHeld]
+    )
+  );
+
+  useEventTarget(
+    window,
+    "blur",
+    useCallback(() => {
+      if (spacebarHeld) {
+        setSpacebarHeld(false);
+        setMicrophoneMuted(true);
+      }
+    }, [setMicrophoneMuted, setSpacebarHeld, spacebarHeld])
+  );
+}

src/video-grid/VideoTile.tsx

@@ -36,6 +36,7 @@ interface Props {
   mediaRef?: React.RefObject<MediaElement>;
   onOptionsPress?: () => void;
   localVolume?: number;
+  hasAudio?: boolean;
   maximised?: boolean;
   fullscreen?: boolean;
   onFullscreen?: () => void;
@@ -58,6 +59,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       mediaRef,
       onOptionsPress,
       localVolume,
+      hasAudio,
       maximised,
       fullscreen,
       onFullscreen,
@@ -74,14 +76,16 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
 
     const toolbarButtons: JSX.Element[] = [];
     if (connectionState == ConnectionState.Connected && !isLocal) {
-      toolbarButtons.push(
-        <AudioButton
-          key="localVolume"
-          className={styles.button}
-          volume={localVolume}
-          onPress={onOptionsPress}
-        />
-      );
+      if (hasAudio) {
+        toolbarButtons.push(
+          <AudioButton
+            key="localVolume"
+            className={styles.button}
+            volume={localVolume}
+            onPress={onOptionsPress}
+          />
+        );
+      }
 
       if (screenshare) {
         toolbarButtons.push(
@@ -137,7 +141,13 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             </div>
           ) : (
             <div className={classNames(styles.infoBubble, styles.memberName)}>
-              {audioMuted && !videoMuted && <MicMutedIcon />}
+              {
+                /* If the user is speaking, it's safe to say they're unmuted.
+                Mute state is currently sent over to-device messages, which
+                aren't quite real-time, so this is an important kludge to make
+                sure no one appears muted when they've clearly begun talking. */
+                audioMuted && !videoMuted && !speaking && <MicMutedIcon />
+              }
               {videoMuted && <VideoMutedIcon />}
               <span title={caption}>{caption}</span>
             </div>

src/video-grid/VideoTileContainer.tsx

@@ -62,6 +62,7 @@ export function VideoTileContainer({
     audioMuted,
     videoMuted,
     localVolume,
+    hasAudio,
     speaking,
     stream,
     purpose,
@@ -117,6 +118,7 @@ export function VideoTileContainer({
         avatar={getAvatar && getAvatar(item.member, width, height)}
         onOptionsPress={onOptionsPress}
         localVolume={localVolume}
+        hasAudio={hasAudio}
         maximised={maximised}
         fullscreen={fullscreen}
         onFullscreen={onFullscreenCallback}

src/video-grid/useCallFeed.ts

@@ -25,6 +25,7 @@ interface CallFeedState {
   videoMuted: boolean;
   audioMuted: boolean;
   localVolume: number;
+  hasAudio: boolean;
   disposed: boolean | undefined;
   stream: MediaStream | undefined;
   purpose: SDPStreamMetadataPurpose | undefined;
@@ -38,6 +39,7 @@ function getCallFeedState(callFeed: CallFeed | undefined): CallFeedState {
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,
     audioMuted: callFeed ? callFeed.isAudioMuted() : true,
     localVolume: callFeed ? callFeed.getLocalVolume() : 0,
+    hasAudio: callFeed ? callFeed.stream.getAudioTracks().length >= 1 : false,
     disposed: callFeed ? callFeed.disposed : undefined,
     stream: callFeed ? callFeed.stream : undefined,
     purpose: callFeed ? callFeed.purpose : undefined,

yarn.lock

@@ -10362,9 +10362,9 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#76075e050f859d668218dcb6f8a2e6940f23fa4e":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#e1a14812fd9f0a261b0e823e3cc68d9ee93b8f11":
   version "23.0.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/76075e050f859d668218dcb6f8a2e6940f23fa4e"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/e1a14812fd9f0a261b0e823e3cc68d9ee93b8f11"
   dependencies:
     "@babel/runtime" "^7.12.5"
     "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.2"
@@ -10375,7 +10375,6 @@ matrix-events-sdk@0.0.1:
     matrix-events-sdk "0.0.1"
     matrix-widget-api "^1.0.0"
     p-retry "4"
-    qs "^6.9.6"
     sdp-transform "^2.14.1"
     unhomoglyph "^1.0.6"
     uuid "9"
@@ -12100,7 +12099,7 @@ qs@6.10.3:
   dependencies:
     side-channel "^1.0.4"
 
-qs@^6.10.0, qs@^6.9.6:
+qs@^6.10.0:
   version "6.11.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a"
   integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==