Merge pull request #319 from vector-im/dbkr/pttsounds

Add push-to-talk sound effects

.eslintrc.js

@@ -19,6 +19,7 @@ module.exports = {
         // We break this rule in a few places: dial it back to a warning
         // (and run with max warnings) to tolerate the existing code
         "react-hooks/exhaustive-deps": ["warn"],
+        "jsx-a11y/media-has-caption": ["off"],
     },
     overrides: [
         {

src/room/PTTCallView.tsx

@@ -17,7 +17,6 @@ limitations under the License.
 import React from "react";
 import useMeasure from "react-use-measure";
 import { ResizeObserver } from "@juggle/resize-observer";
-import { OtherUserSpeakingError } from "matrix-js-sdk/src/webrtc/groupCall";
 
 import { useModalTriggerState } from "../Modal";
 import { SettingsModal } from "../settings/SettingsModal";
@@ -34,6 +33,8 @@ import { Timer } from "./Timer";
 import { Toggle } from "../input/Toggle";
 import { getAvatarUrl } from "../matrix-utils";
 import { ReactComponent as AudioIcon } from "../icons/Audio.svg";
+import { usePTTSounds } from "../sound/usePttSounds";
+import { PTTClips } from "../sound/PTTClips";
 
 export function PTTCallView({
   client,
@@ -57,6 +58,9 @@ export function PTTCallView({
 
   const { audioOutput } = useMediaHandler();
 
+  const { startTalkingLocalRef, startTalkingRemoteRef, blockedRef, playClip } =
+    usePTTSounds();
+
   const {
     pttButtonHeld,
     isAdmin,
@@ -65,11 +69,10 @@ export function PTTCallView({
     activeSpeakerUserId,
     startTalking,
     stopTalking,
-    unmuteError,
-  } = usePTT(client, groupCall, userMediaFeeds);
+    transmitBlocked,
+  } = usePTT(client, groupCall, userMediaFeeds, playClip);
 
-  const showTalkOverError =
-    pttButtonHeld && unmuteError instanceof OtherUserSpeakingError;
+  const showTalkOverError = pttButtonHeld && transmitBlocked;
 
   const activeSpeakerIsLocalUser =
     activeSpeakerUserId && client.getUserId() === activeSpeakerUserId;
@@ -89,6 +92,11 @@ export function PTTCallView({
 
   return (
     <div className={styles.pttCallView} ref={containerRef}>
+      <PTTClips
+        startTalkingLocalRef={startTalkingLocalRef}
+        startTalkingRemoteRef={startTalkingRemoteRef}
+        blockedRef={blockedRef}
+      />
       <Header className={styles.header}>
         <LeftNav>
           <RoomSetupHeaderInfo roomName={roomName} onPress={onLeave} />

src/room/usePTT.ts

@@ -18,6 +18,9 @@ import { useCallback, useEffect, useState } from "react";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
+import { logger } from "matrix-js-sdk/src/logger";
+
+import { PlayClipFunction, PTTClipID } from "../sound/usePttSounds";
 
 export interface PTTState {
   pttButtonHeld: boolean;
@@ -27,13 +30,14 @@ export interface PTTState {
   activeSpeakerUserId: string;
   startTalking: () => void;
   stopTalking: () => void;
-  unmuteError: Error;
+  transmitBlocked: boolean;
 }
 
 export const usePTT = (
   client: MatrixClient,
   groupCall: GroupCall,
-  userMediaFeeds: CallFeed[]
+  userMediaFeeds: CallFeed[],
+  playClip: PlayClipFunction
 ): PTTState => {
   const [
     {
@@ -41,7 +45,7 @@ export const usePTT = (
       isAdmin,
       talkOverEnabled,
       activeSpeakerUserId,
-      unmuteError,
+      transmitBlocked,
     },
     setState,
   ] = useState(() => {
@@ -54,7 +58,7 @@ export const usePTT = (
       talkOverEnabled: false,
       pttButtonHeld: false,
       activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null,
-      unmuteError: null,
+      transmitBlocked: false,
     };
   });
 
@@ -62,6 +66,21 @@ export const usePTT = (
     function onMuteStateChanged(...args): void {
       const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted());
 
+      if (activeSpeakerUserId === null && activeSpeakerFeed.userId !== null) {
+        if (activeSpeakerFeed.userId === client.getUserId()) {
+          playClip(PTTClipID.START_TALKING_LOCAL);
+        } else {
+          playClip(PTTClipID.START_TALKING_REMOTE);
+        }
+      } else if (
+        activeSpeakerFeed &&
+        activeSpeakerUserId === client.getUserId() &&
+        activeSpeakerFeed.userId !== client.getUserId()
+      ) {
+        // We were talking but we've been cut off
+        playClip(PTTClipID.BLOCKED);
+      }
+
       setState((prevState) => ({
         ...prevState,
         activeSpeakerUserId: activeSpeakerFeed
@@ -89,33 +108,55 @@ export const usePTT = (
         );
       }
     };
-  }, [userMediaFeeds]);
+  }, [userMediaFeeds, activeSpeakerUserId, client, playClip]);
 
   const startTalking = useCallback(async () => {
-    setState((prevState) => ({
-      ...prevState,
-      pttButtonHeld: true,
-      unmuteError: null,
-    }));
-    if (!activeSpeakerUserId || isAdmin || talkOverEnabled) {
+    if (pttButtonHeld) return;
+
+    let blocked = false;
+    if (!activeSpeakerUserId || (isAdmin && talkOverEnabled)) {
       if (groupCall.isMicrophoneMuted()) {
         try {
           await groupCall.setMicrophoneMuted(false);
         } catch (e) {
-          setState((prevState) => ({ ...prevState, unmuteError: null }));
+          logger.error("Failed to unmute microphone", e);
         }
       }
+    } else {
+      playClip(PTTClipID.BLOCKED);
+      blocked = true;
     }
-  }, [groupCall, activeSpeakerUserId, isAdmin, talkOverEnabled, setState]);
+    setState((prevState) => ({
+      ...prevState,
+      pttButtonHeld: true,
+      transmitBlocked: blocked,
+    }));
+  }, [
+    pttButtonHeld,
+    groupCall,
+    activeSpeakerUserId,
+    isAdmin,
+    talkOverEnabled,
+    setState,
+    playClip,
+  ]);
 
   const stopTalking = useCallback(() => {
-    setState((prevState) => ({ ...prevState, pttButtonHeld: false }));
+    setState((prevState) => ({
+      ...prevState,
+      pttButtonHeld: false,
+      unmuteError: null,
+    }));
 
     if (!groupCall.isMicrophoneMuted()) {
       groupCall.setMicrophoneMuted(true);
     }
 
-    setState((prevState) => ({ ...prevState, pttButtonHeld: false }));
+    setState((prevState) => ({
+      ...prevState,
+      pttButtonHeld: false,
+      transmitBlocked: false,
+    }));
   }, [groupCall]);
 
   useEffect(() => {
@@ -180,6 +221,6 @@ export const usePTT = (
     activeSpeakerUserId,
     startTalking,
     stopTalking,
-    unmuteError,
+    transmitBlocked,
   };
 };

src/sound/PTTClips.module.css

@@ -0,0 +1,19 @@
+/*
+Copyright 2022 Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.pttClip {
+  display: none;
+}

src/sound/PTTClips.tsx

@@ -0,0 +1,62 @@
+/*
+Copyright 2022 Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React from "react";
+
+import startTalkLocalOggUrl from "./start_talk_local.ogg";
+import startTalkLocalMp3Url from "./start_talk_local.mp3";
+import startTalkRemoteOggUrl from "./start_talk_remote.ogg";
+import startTalkRemoteMp3Url from "./start_talk_remote.mp3";
+import blockedOggUrl from "./blocked.ogg";
+import blockedMp3Url from "./blocked.mp3";
+import styles from "./PTTClips.module.css";
+
+interface Props {
+  startTalkingLocalRef: React.RefObject<HTMLAudioElement>;
+  startTalkingRemoteRef: React.RefObject<HTMLAudioElement>;
+  blockedRef: React.RefObject<HTMLAudioElement>;
+}
+
+export const PTTClips: React.FC<Props> = ({
+  startTalkingLocalRef,
+  startTalkingRemoteRef,
+  blockedRef,
+}) => {
+  return (
+    <>
+      <audio
+        preload="true"
+        className={styles.pttClip}
+        ref={startTalkingLocalRef}
+      >
+        <source type="audio/ogg" src={startTalkLocalOggUrl} />
+        <source type="audio/mpeg" src={startTalkLocalMp3Url} />
+      </audio>
+      <audio
+        preload="true"
+        className={styles.pttClip}
+        ref={startTalkingRemoteRef}
+      >
+        <source type="audio/ogg" src={startTalkRemoteOggUrl} />
+        <source type="audio/mpeg" src={startTalkRemoteMp3Url} />
+      </audio>
+      <audio preload="true" className={styles.pttClip} ref={blockedRef}>
+        <source type="audio/ogg" src={blockedOggUrl} />
+        <source type="audio/mpeg" src={blockedMp3Url} />
+      </audio>
+    </>
+  );
+};

src/sound/usePttSounds.ts

@@ -0,0 +1,70 @@
+/*
+Copyright 2022 Matrix.org Foundation C.I.C.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, { useCallback, useState } from "react";
+
+export enum PTTClipID {
+  START_TALKING_LOCAL,
+  START_TALKING_REMOTE,
+  BLOCKED,
+}
+
+export type PlayClipFunction = (clipID: PTTClipID) => void;
+
+interface PTTSounds {
+  startTalkingLocalRef: React.RefObject<HTMLAudioElement>;
+  startTalkingRemoteRef: React.RefObject<HTMLAudioElement>;
+  blockedRef: React.RefObject<HTMLAudioElement>;
+  playClip: PlayClipFunction;
+}
+
+export const usePTTSounds = (): PTTSounds => {
+  const [startTalkingLocalRef] = useState(React.createRef<HTMLAudioElement>());
+  const [startTalkingRemoteRef] = useState(React.createRef<HTMLAudioElement>());
+  const [blockedRef] = useState(React.createRef<HTMLAudioElement>());
+
+  const playClip = useCallback(
+    async (clipID: PTTClipID) => {
+      let ref: React.RefObject<HTMLAudioElement>;
+
+      switch (clipID) {
+        case PTTClipID.START_TALKING_LOCAL:
+          ref = startTalkingLocalRef;
+          break;
+        case PTTClipID.START_TALKING_REMOTE:
+          ref = startTalkingRemoteRef;
+          break;
+        case PTTClipID.BLOCKED:
+          ref = blockedRef;
+          break;
+      }
+      if (ref.current) {
+        ref.current.currentTime = 0;
+        await ref.current.play();
+      } else {
+        console.log("No media element found");
+      }
+    },
+    [startTalkingLocalRef, startTalkingRemoteRef, blockedRef]
+  );
+
+  return {
+    startTalkingLocalRef,
+    startTalkingRemoteRef,
+    blockedRef,
+    playClip,
+  };
+};