Merge remote-tracking branch 'origin/main' into dbkr/rageshake_ptt

package.json

@@ -13,6 +13,7 @@
   },
   "dependencies": {
     "@juggle/resize-observer": "^3.3.1",
+    "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz",
     "@react-aria/button": "^3.3.4",
     "@react-aria/dialog": "^3.1.4",
     "@react-aria/focus": "^3.5.0",

src/IndexedDBWorker.js

@@ -0,0 +1,5 @@
+import { IndexedDBStoreWorker } from "matrix-js-sdk/src/indexeddb-worker";
+
+const remoteWorker = new IndexedDBStoreWorker(self.postMessage);
+
+self.onmessage = remoteWorker.onMessage;

src/matrix-utils.js

@@ -3,6 +3,9 @@ import {
   GroupCallIntent,
   GroupCallType,
 } from "matrix-js-sdk/src/browser-index";
+import IndexedDBWorker from "./IndexedDBWorker?worker";
+import Olm from "@matrix-org/olm";
+import olmWasmPath from "@matrix-org/olm/olm.wasm?url";
 
 export const defaultHomeserver =
   import.meta.env.VITE_DEFAULT_HOMESERVER ||
@@ -26,11 +29,59 @@ function waitForSync(client) {
 }
 
 export async function initClient(clientOptions) {
+  // TODO: https://gitlab.matrix.org/matrix-org/olm/-/issues/10
+  window.OLM_OPTIONS = {};
+  await Olm.init({ locateFile: () => olmWasmPath });
+
+  let indexedDB;
+
+  try {
+    indexedDB = window.indexedDB;
+  } catch (e) {}
+
+  const storeOpts = {};
+
+  if (indexedDB && localStorage && !import.meta.env.DEV) {
+    storeOpts.store = new matrix.IndexedDBStore({
+      indexedDB: window.indexedDB,
+      localStorage: window.localStorage,
+      dbName: "element-call-sync",
+      workerFactory: () => new IndexedDBWorker(),
+    });
+  }
+
+  if (localStorage) {
+    storeOpts.sessionStore = new matrix.WebStorageSessionStore(localStorage);
+  }
+
+  if (indexedDB) {
+    storeOpts.cryptoStore = new matrix.IndexedDBCryptoStore(
+      indexedDB,
+      "matrix-js-sdk:crypto"
+    );
+  }
+
   const client = matrix.createClient({
+    ...storeOpts,
     ...clientOptions,
     useAuthorizationHeader: true,
   });
 
+  try {
+    await client.store.startup();
+  } catch (error) {
+    console.error(
+      "Error starting matrix client store. Falling back to memory store.",
+      error
+    );
+    client.store = new matrix.MemoryStore({ localStorage });
+    await client.store.startup();
+  }
+
+  if (client.initCrypto) {
+    await client.initCrypto();
+  }
+
   await client.startClient({
     // dirty hack to reduce chance of gappy syncs
     // should be fixed by spotting gaps and backpaginating

src/room/PTTCallView.tsx

@@ -110,8 +110,13 @@ export const PTTCallView: React.FC<Props> = ({
 
   const { audioOutput } = useMediaHandler();
 
-  const { startTalkingLocalRef, startTalkingRemoteRef, blockedRef, playClip } =
+  const {
-    usePTTSounds();
+    startTalkingLocalRef,
+    startTalkingRemoteRef,
+    blockedRef,
+    endTalkingRef,
+    playClip,
+  } = usePTTSounds();
 
   const {
     pttButtonHeld,
@@ -153,6 +158,7 @@ export const PTTCallView: React.FC<Props> = ({
       <PTTClips
         startTalkingLocalRef={startTalkingLocalRef}
         startTalkingRemoteRef={startTalkingRemoteRef}
+        endTalkingRef={endTalkingRef}
         blockedRef={blockedRef}
       />
       <GroupCallInspector

src/room/usePTT.ts

@@ -22,6 +22,27 @@ import { logger } from "matrix-js-sdk/src/logger";
 
 import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
 
+// Works out who the active speaker should be given what feeds are active and
+// the power level of each user.
+function getActiveSpeakerFeed(
+  feeds: CallFeed[],
+  groupCall: GroupCall
+): CallFeed | null {
+  const activeSpeakerFeeds = feeds.filter((f) => !f.isAudioMuted());
+
+  let activeSpeakerFeed = null;
+  let highestPowerLevel = null;
+  for (const feed of activeSpeakerFeeds) {
+    const member = groupCall.room.getMember(feed.userId);
+    if (highestPowerLevel === null || member.powerLevel > highestPowerLevel) {
+      highestPowerLevel = member.powerLevel;
+      activeSpeakerFeed = feed;
+    }
+  }
+
+  return activeSpeakerFeed;
+}
+
 export interface PTTState {
   pttButtonHeld: boolean;
   isAdmin: boolean;
@@ -40,6 +61,26 @@ export const usePTT = (
   playClip: PlayClipFunction,
   enablePTTButton: boolean
 ): PTTState => {
+  // Used to serialise all the mute calls so they don't race. It has
+  // its own state as its always set separately from anything else.
+  const [mutePromise, setMutePromise] = useState(
+    Promise.resolve<boolean | void>(false)
+  );
+
+  // Wrapper to serialise all the mute operations on the promise
+  const setMicMuteWrapper = useCallback(
+    (muted: boolean) => {
+      setMutePromise(
+        mutePromise.then(() => {
+          return groupCall.setMicrophoneMuted(muted).catch((e) => {
+            logger.error("Failed to unmute microphone", e);
+          });
+        })
+      );
+    },
+    [groupCall, mutePromise]
+  );
+
   const [
     {
       pttButtonHeld,
@@ -52,7 +93,7 @@ export const usePTT = (
   ] = useState(() => {
     const roomMember = groupCall.room.getMember(client.getUserId());
 
-    const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted());
+    const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
 
     return {
       isAdmin: roomMember.powerLevel >= 100,
@@ -63,38 +104,58 @@ export const usePTT = (
     };
   });
 
-  useEffect(() => {
+  const onMuteStateChanged = useCallback(() => {
-    function onMuteStateChanged(...args): void {
+    const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
-      const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted());
 
-      if (activeSpeakerUserId === null && activeSpeakerFeed.userId !== null) {
+    let blocked = false;
+    if (activeSpeakerUserId === null && activeSpeakerFeed !== null) {
       if (activeSpeakerFeed.userId === client.getUserId()) {
         playClip(PTTClipID.START_TALKING_LOCAL);
       } else {
         playClip(PTTClipID.START_TALKING_REMOTE);
       }
+    } else if (activeSpeakerUserId !== null && activeSpeakerFeed === null) {
+      playClip(PTTClipID.END_TALKING);
     } else if (
       pttButtonHeld &&
       activeSpeakerUserId === client.getUserId() &&
       activeSpeakerFeed?.userId !== client.getUserId()
     ) {
-        // We were talking but we've been cut off
+      // We were talking but we've been cut off: mute our own mic
+      // (this is the easier way of cutting other speakers off if an
+      // admin barges in: we could also mute the non-admin speaker
+      // on all receivers, but we'd have to make sure we unmuted them
+      // correctly.)
+      setMicMuteWrapper(true);
+      blocked = true;
       playClip(PTTClipID.BLOCKED);
     }
 
-      setState((prevState) => ({
+    setState((prevState) => {
+      return {
         ...prevState,
         activeSpeakerUserId: activeSpeakerFeed
           ? activeSpeakerFeed.userId
           : null,
-      }));
+        transmitBlocked: blocked,
-    }
+      };
+    });
+  }, [
+    playClip,
+    groupCall,
+    pttButtonHeld,
+    activeSpeakerUserId,
+    client,
+    userMediaFeeds,
+    setMicMuteWrapper,
+  ]);
 
+  useEffect(() => {
     for (const callFeed of userMediaFeeds) {
       callFeed.addListener(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
     }
 
-    const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted());
+    const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall);
 
     setState((prevState) => ({
       ...prevState,
@@ -109,29 +170,26 @@ export const usePTT = (
         );
       }
     };
-  }, [userMediaFeeds, activeSpeakerUserId, client, playClip, pttButtonHeld]);
+  }, [userMediaFeeds, onMuteStateChanged, groupCall]);
 
   const startTalking = useCallback(async () => {
     if (pttButtonHeld) return;
 
     let blocked = false;
-    if (!activeSpeakerUserId || (isAdmin && talkOverEnabled)) {
+    if (activeSpeakerUserId && !(isAdmin && talkOverEnabled)) {
-      if (groupCall.isMicrophoneMuted()) {
-        try {
-          await groupCall.setMicrophoneMuted(false);
-        } catch (e) {
-          logger.error("Failed to unmute microphone", e);
-        }
-      }
-    } else {
       playClip(PTTClipID.BLOCKED);
       blocked = true;
     }
+    // setstate before doing the async call to mute / unmute the mic
     setState((prevState) => ({
       ...prevState,
       pttButtonHeld: true,
       transmitBlocked: blocked,
     }));
+
+    if (!blocked && groupCall.isMicrophoneMuted()) {
+      setMicMuteWrapper(false);
+    }
   }, [
     pttButtonHeld,
     groupCall,
@@ -140,25 +198,18 @@ export const usePTT = (
     talkOverEnabled,
     setState,
     playClip,
+    setMicMuteWrapper,
   ]);
 
-  const stopTalking = useCallback(() => {
+  const stopTalking = useCallback(async () => {
-    setState((prevState) => ({
-      ...prevState,
-      pttButtonHeld: false,
-      unmuteError: null,
-    }));
-
-    if (!groupCall.isMicrophoneMuted()) {
-      groupCall.setMicrophoneMuted(true);
-    }
-
     setState((prevState) => ({
       ...prevState,
       pttButtonHeld: false,
       transmitBlocked: false,
     }));
-  }, [groupCall]);
+
+    setMicMuteWrapper(true);
+  }, [setMicMuteWrapper]);
 
   useEffect(() => {
     function onKeyDown(event: KeyboardEvent): void {
@@ -184,7 +235,7 @@ export const usePTT = (
     function onBlur(): void {
       // TODO: We will need to disable this for a global PTT hotkey to work
       if (!groupCall.isMicrophoneMuted()) {
-        groupCall.setMicrophoneMuted(true);
+        setMicMuteWrapper(true);
       }
 
       setState((prevState) => ({ ...prevState, pttButtonHeld: false }));
@@ -208,6 +259,7 @@ export const usePTT = (
     talkOverEnabled,
     pttButtonHeld,
     enablePTTButton,
+    setMicMuteWrapper,
   ]);
 
   const setTalkOverEnabled = useCallback((talkOverEnabled) => {

src/sound/PTTClips.tsx

@@ -20,6 +20,8 @@ import startTalkLocalOggUrl from "./start_talk_local.ogg";
 import startTalkLocalMp3Url from "./start_talk_local.mp3";
 import startTalkRemoteOggUrl from "./start_talk_remote.ogg";
 import startTalkRemoteMp3Url from "./start_talk_remote.mp3";
+import endTalkOggUrl from "./end_talk.ogg";
+import endTalkMp3Url from "./end_talk.mp3";
 import blockedOggUrl from "./blocked.ogg";
 import blockedMp3Url from "./blocked.mp3";
 import styles from "./PTTClips.module.css";
@@ -27,12 +29,14 @@ import styles from "./PTTClips.module.css";
 interface Props {
   startTalkingLocalRef: React.RefObject<HTMLAudioElement>;
   startTalkingRemoteRef: React.RefObject<HTMLAudioElement>;
+  endTalkingRef: React.RefObject<HTMLAudioElement>;
   blockedRef: React.RefObject<HTMLAudioElement>;
 }
 
 export const PTTClips: React.FC<Props> = ({
   startTalkingLocalRef,
   startTalkingRemoteRef,
+  endTalkingRef,
   blockedRef,
 }) => {
   return (
@@ -53,6 +57,10 @@ export const PTTClips: React.FC<Props> = ({
         <source type="audio/ogg" src={startTalkRemoteOggUrl} />
         <source type="audio/mpeg" src={startTalkRemoteMp3Url} />
       </audio>
+      <audio preload="true" className={styles.pttClip} ref={endTalkingRef}>
+        <source type="audio/ogg" src={endTalkOggUrl} />
+        <source type="audio/mpeg" src={endTalkMp3Url} />
+      </audio>
       <audio preload="true" className={styles.pttClip} ref={blockedRef}>
         <source type="audio/ogg" src={blockedOggUrl} />
         <source type="audio/mpeg" src={blockedMp3Url} />

src/sound/usePttSounds.ts

@@ -19,6 +19,7 @@ import React, { useCallback, useState } from "react";
 export enum PTTClipID {
   START_TALKING_LOCAL,
   START_TALKING_REMOTE,
+  END_TALKING,
   BLOCKED,
 }
 
@@ -27,6 +28,7 @@ export type PlayClipFunction = (clipID: PTTClipID) => void;
 interface PTTSounds {
   startTalkingLocalRef: React.RefObject<HTMLAudioElement>;
   startTalkingRemoteRef: React.RefObject<HTMLAudioElement>;
+  endTalkingRef: React.RefObject<HTMLAudioElement>;
   blockedRef: React.RefObject<HTMLAudioElement>;
   playClip: PlayClipFunction;
 }
@@ -34,6 +36,7 @@ interface PTTSounds {
 export const usePTTSounds = (): PTTSounds => {
   const [startTalkingLocalRef] = useState(React.createRef<HTMLAudioElement>());
   const [startTalkingRemoteRef] = useState(React.createRef<HTMLAudioElement>());
+  const [endTalkingRef] = useState(React.createRef<HTMLAudioElement>());
   const [blockedRef] = useState(React.createRef<HTMLAudioElement>());
 
   const playClip = useCallback(
@@ -47,6 +50,9 @@ export const usePTTSounds = (): PTTSounds => {
         case PTTClipID.START_TALKING_REMOTE:
           ref = startTalkingRemoteRef;
           break;
+        case PTTClipID.END_TALKING:
+          ref = endTalkingRef;
+          break;
         case PTTClipID.BLOCKED:
           ref = blockedRef;
           break;
@@ -58,12 +64,13 @@ export const usePTTSounds = (): PTTSounds => {
         console.log("No media element found");
       }
     },
-    [startTalkingLocalRef, startTalkingRemoteRef, blockedRef]
+    [startTalkingLocalRef, startTalkingRemoteRef, endTalkingRef, blockedRef]
   );
 
   return {
     startTalkingLocalRef,
     startTalkingRemoteRef,
+    endTalkingRef,
     blockedRef,
     playClip,
   };