Merge branch 'main' into chrome-spatial-aec

.env

@@ -22,6 +22,7 @@
 # VITE_THEME_PRIMARY_CONTENT=#ffffff
 # VITE_THEME_SECONDARY_CONTENT=#a9b2bc
 # VITE_THEME_TERTIARY_CONTENT=#8e99a4
+# VITE_THEME_TERTIARY_CONTENT_20=#8e99a433
 # VITE_THEME_QUATERNARY_CONTENT=#6f7882
 # VITE_THEME_QUINARY_CONTENT=#394049
 # VITE_THEME_SYSTEM=#21262c

src/index.css

@@ -33,6 +33,7 @@ limitations under the License.
   --primary-content: #ffffff;
   --secondary-content: #a9b2bc;
   --tertiary-content: #8e99a4;
+  --tertiary-content-20: #8e99a433;
   --quaternary-content: #6f7882;
   --quinary-content: #394049;
   --system: #21262c;

src/main.tsx

@@ -61,6 +61,10 @@ if (import.meta.env.VITE_CUSTOM_THEME) {
     "--tertiary-content",
     import.meta.env.VITE_THEME_TERTIARY_CONTENT as string
   );
+  style.setProperty(
+    "--tertiary-content-20",
+    import.meta.env.VITE_THEME_TERTIARY_CONTENT_20 as string
+  );
   style.setProperty(
     "--quaternary-content",
     import.meta.env.VITE_THEME_QUATERNARY_CONTENT as string

src/room/PTTButton.module.css

@@ -17,6 +17,12 @@
   cursor: unset;
 }
 
+.networkWaiting {
+  background-color: var(--tertiary-content);
+  border-color: var(--tertiary-content);
+  cursor: unset;
+}
+
 .error {
   background-color: var(--alert);
   border-color: var(--alert);

src/room/PTTButton.tsx

@@ -14,15 +14,17 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useCallback, useEffect, useState, createRef } from "react";
+import React, { useCallback, useState, createRef } from "react";
 import classNames from "classnames";
 import { useSpring, animated } from "@react-spring/web";
 
 import styles from "./PTTButton.module.css";
 import { ReactComponent as MicIcon } from "../icons/Mic.svg";
+import { useEventTarget } from "../useEvents";
 import { Avatar } from "../Avatar";
 
 interface Props {
+  enabled: boolean;
   showTalkOverError: boolean;
   activeSpeakerUserId: string;
   activeSpeakerDisplayName: string;
@@ -32,15 +34,13 @@ interface Props {
   size: number;
   startTalking: () => void;
   stopTalking: () => void;
-}
+  networkWaiting: boolean;
-
+  enqueueNetworkWaiting: (value: boolean, delay: number) => void;
-interface State {
+  setNetworkWaiting: (value: boolean) => void;
-  isHeld: boolean;
-  // If the button is being pressed by touch, the ID of that touch
-  activeTouchID: number | null;
 }
 
 export const PTTButton: React.FC<Props> = ({
+  enabled,
   showTalkOverError,
   activeSpeakerUserId,
   activeSpeakerDisplayName,
@@ -50,89 +50,110 @@ export const PTTButton: React.FC<Props> = ({
   size,
   startTalking,
   stopTalking,
+  networkWaiting,
+  enqueueNetworkWaiting,
+  setNetworkWaiting,
 }) => {
   const buttonRef = createRef<HTMLButtonElement>();
 
-  const [{ isHeld, activeTouchID }, setState] = useState<State>({
+  const [activeTouchId, setActiveTouchId] = useState<number | null>(null);
-    isHeld: false,
-    activeTouchID: null,
-  });
-  const onWindowMouseUp = useCallback(
-    (e) => {
-      if (isHeld) stopTalking();
-      setState({ isHeld: false, activeTouchID: null });
-    },
-    [isHeld, setState, stopTalking]
-  );
 
-  const onWindowTouchEnd = useCallback(
+  const hold = useCallback(() => {
-    (e: TouchEvent) => {
+    // This update is delayed so the user only sees it if latency is significant
-      // ignore any ended touches that weren't the one pressing the
+    enqueueNetworkWaiting(true, 100);
-      // button (bafflingly the TouchList isn't an iterable so we
+    startTalking();
-      // have to do this a really old-school way).
+  }, [enqueueNetworkWaiting, startTalking]);
-      let touchFound = false;
+  const unhold = useCallback(() => {
-      for (let i = 0; i < e.changedTouches.length; ++i) {
+    setNetworkWaiting(false);
-        if (e.changedTouches.item(i).identifier === activeTouchID) {
+    stopTalking();
-          touchFound = true;
+  }, [setNetworkWaiting, stopTalking]);
-          break;
-        }
-      }
-      if (!touchFound) return;
-
-      e.preventDefault();
-      if (isHeld) stopTalking();
-      setState({ isHeld: false, activeTouchID: null });
-    },
-    [isHeld, activeTouchID, setState, stopTalking]
-  );
 
   const onButtonMouseDown = useCallback(
     (e: React.MouseEvent<HTMLButtonElement>) => {
       e.preventDefault();
-      setState({ isHeld: true, activeTouchID: null });
+      hold();
-      startTalking();
     },
-    [setState, startTalking]
+    [hold]
   );
 
-  const onButtonTouchStart = useCallback(
+  // These listeners go on the window so even if the user's cursor / finger
-    (e: TouchEvent) => {
+  // leaves the button while holding it, the button stays pushed until
-      e.preventDefault();
+  // they stop clicking / tapping.
+  useEventTarget(window, "mouseup", unhold);
+  useEventTarget(
+    window,
+    "touchend",
+    useCallback(
+      (e: TouchEvent) => {
+        // ignore any ended touches that weren't the one pressing the
+        // button (bafflingly the TouchList isn't an iterable so we
+        // have to do this a really old-school way).
+        let touchFound = false;
+        for (let i = 0; i < e.changedTouches.length; ++i) {
+          if (e.changedTouches.item(i).identifier === activeTouchId) {
+            touchFound = true;
+            break;
+          }
+        }
+        if (!touchFound) return;
 
-      if (isHeld) return;
+        e.preventDefault();
-
+        unhold();
-      setState({
+        setActiveTouchId(null);
-        isHeld: true,
+      },
-        activeTouchID: e.changedTouches.item(0).identifier,
+      [unhold, activeTouchId, setActiveTouchId]
-      });
+    )
-      startTalking();
-    },
-    [isHeld, setState, startTalking]
   );
 
-  useEffect(() => {
+  // This is a native DOM listener too because we want to preventDefault in it
-    const currentButtonElement = buttonRef.current;
+  // to stop also getting a click event, so we need it to be non-passive.
+  useEventTarget(
+    buttonRef.current,
+    "touchstart",
+    useCallback(
+      (e: TouchEvent) => {
+        e.preventDefault();
 
-    // These listeners go on the window so even if the user's cursor / finger
+        hold();
-    // leaves the button while holding it, the button stays pushed until
+        setActiveTouchId(e.changedTouches.item(0).identifier);
-    // they stop clicking / tapping.
+      },
-    window.addEventListener("mouseup", onWindowMouseUp);
+      [hold, setActiveTouchId]
-    window.addEventListener("touchend", onWindowTouchEnd);
+    ),
-    // This is a native DOM listener too because we want to preventDefault in it
+    { passive: false }
-    // to stop also getting a click event, so we need it to be non-passive.
+  );
-    currentButtonElement.addEventListener("touchstart", onButtonTouchStart, {
-      passive: false,
-    });
 
-    return () => {
+  useEventTarget(
-      window.removeEventListener("mouseup", onWindowMouseUp);
+    window,
-      window.removeEventListener("touchend", onWindowTouchEnd);
+    "keydown",
-      currentButtonElement.removeEventListener(
+    useCallback(
-        "touchstart",
+      (e: KeyboardEvent) => {
-        onButtonTouchStart
+        if (e.code === "Space") {
-      );
+          if (!enabled) return;
-    };
+          e.preventDefault();
-  }, [onWindowMouseUp, onWindowTouchEnd, onButtonTouchStart, buttonRef]);
+
+          hold();
+        }
+      },
+      [enabled, hold]
+    )
+  );
+  useEventTarget(
+    window,
+    "keyup",
+    useCallback(
+      (e: KeyboardEvent) => {
+        if (e.code === "Space") {
+          e.preventDefault();
+
+          unhold();
+        }
+      },
+      [unhold]
+    )
+  );
+
+  // TODO: We will need to disable this for a global PTT hotkey to work
+  useEventTarget(window, "blur", unhold);
 
   const { shadow } = useSpring({
     shadow: (Math.max(activeSpeakerVolume, -70) + 70) * 0.6,
@@ -143,12 +164,15 @@ export const PTTButton: React.FC<Props> = ({
   });
   const shadowColor = showTalkOverError
     ? "var(--alert-20)"
+    : networkWaiting
+    ? "var(--tertiary-content-20)"
     : "var(--accent-20)";
 
   return (
     <animated.button
       className={classNames(styles.pttButton, {
         [styles.talking]: activeSpeakerUserId,
+        [styles.networkWaiting]: networkWaiting,
         [styles.error]: showTalkOverError,
       })}
       style={{

src/room/PTTCallView.tsx

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React from "react";
+import React, { useEffect } from "react";
 import useMeasure from "react-use-measure";
 import { ResizeObserver } from "@juggle/resize-observer";
 import { GroupCall, MatrixClient, RoomMember } from "matrix-js-sdk";
 import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
 
+import { useDelayedState } from "../useDelayedState";
 import { useModalTriggerState } from "../Modal";
 import { InviteModal } from "./InviteModal";
 import { HangupButton, InviteButton } from "../button";
@@ -39,6 +40,7 @@ import { GroupCallInspector } from "./GroupCallInspector";
 import { OverflowMenu } from "./OverflowMenu";
 
 function getPromptText(
+  networkWaiting: boolean,
   showTalkOverError: boolean,
   pttButtonHeld: boolean,
   activeSpeakerIsLocalUser: boolean,
@@ -47,10 +49,14 @@ function getPromptText(
   activeSpeakerDisplayName: string,
   connected: boolean
 ): string {
-  if (!connected) return "Connection Lost";
+  if (!connected) return "Connection lost";
 
   const isTouchScreen = Boolean(window.ontouchstart !== undefined);
 
+  if (networkWaiting) {
+    return "Waiting for network";
+  }
+
   if (showTalkOverError) {
     return "You can't talk at the same time";
   }
@@ -128,18 +134,15 @@ export const PTTCallView: React.FC<Props> = ({
     stopTalking,
     transmitBlocked,
     connected,
-  } = usePTT(
+  } = usePTT(client, groupCall, userMediaFeeds, playClip);
-    client,
-    groupCall,
-    userMediaFeeds,
-    playClip,
-    !feedbackModalState.isOpen
-  );
 
+  const [talkingExpected, enqueueTalkingExpected, setTalkingExpected] =
+    useDelayedState(false);
   const showTalkOverError = pttButtonHeld && transmitBlocked;
+  const networkWaiting =
+    talkingExpected && !activeSpeakerUserId && !showTalkOverError;
 
-  const activeSpeakerIsLocalUser =
+  const activeSpeakerIsLocalUser = activeSpeakerUserId === client.getUserId();
-    activeSpeakerUserId && client.getUserId() === activeSpeakerUserId;
   const activeSpeakerUser = activeSpeakerUserId
     ? client.getUser(activeSpeakerUserId)
     : null;
@@ -148,6 +151,10 @@ export const PTTCallView: React.FC<Props> = ({
     ? activeSpeakerUser.displayName
     : "";
 
+  useEffect(() => {
+    setTalkingExpected(activeSpeakerIsLocalUser);
+  }, [activeSpeakerIsLocalUser, setTalkingExpected]);
+
   return (
     <div className={styles.pttCallView} ref={containerRef}>
       <PTTClips
@@ -217,6 +224,7 @@ export const PTTCallView: React.FC<Props> = ({
             <div className={styles.talkingInfo} />
           )}
           <PTTButton
+            enabled={!feedbackModalState.isOpen}
             showTalkOverError={showTalkOverError}
             activeSpeakerUserId={activeSpeakerUserId}
             activeSpeakerDisplayName={activeSpeakerDisplayName}
@@ -226,9 +234,13 @@ export const PTTCallView: React.FC<Props> = ({
             size={pttButtonSize}
             startTalking={startTalking}
             stopTalking={stopTalking}
+            networkWaiting={networkWaiting}
+            enqueueNetworkWaiting={enqueueTalkingExpected}
+            setNetworkWaiting={setTalkingExpected}
           />
           <p className={styles.actionTip}>
             {getPromptText(
+              networkWaiting,
               showTalkOverError,
               pttButtonHeld,
               activeSpeakerIsLocalUser,

src/room/usePTT.ts

@@ -80,8 +80,7 @@ export const usePTT = (
   client: MatrixClient,
   groupCall: GroupCall,
   userMediaFeeds: CallFeed[],
-  playClip: PlayClipFunction,
+  playClip: PlayClipFunction
-  enablePTTButton: boolean
 ): PTTState => {
   // Used to serialise all the mute calls so they don't race. It has
   // its own state as its always set separately from anything else.
@@ -258,59 +257,6 @@ export const usePTT = (
     [setConnected]
   );
 
-  useEffect(() => {
-    function onKeyDown(event: KeyboardEvent): void {
-      if (event.code === "Space") {
-        if (!enablePTTButton) return;
-
-        event.preventDefault();
-
-        if (pttButtonHeld) return;
-
-        startTalking();
-      }
-    }
-
-    function onKeyUp(event: KeyboardEvent): void {
-      if (event.code === "Space") {
-        event.preventDefault();
-
-        stopTalking();
-      }
-    }
-
-    function onBlur(): void {
-      // TODO: We will need to disable this for a global PTT hotkey to work
-      if (!groupCall.isMicrophoneMuted()) {
-        setMicMuteWrapper(true);
-      }
-
-      setState((prevState) => ({ ...prevState, pttButtonHeld: false }));
-    }
-
-    window.addEventListener("keydown", onKeyDown);
-    window.addEventListener("keyup", onKeyUp);
-    window.addEventListener("blur", onBlur);
-
-    return () => {
-      window.removeEventListener("keydown", onKeyDown);
-      window.removeEventListener("keyup", onKeyUp);
-      window.removeEventListener("blur", onBlur);
-    };
-  }, [
-    groupCall,
-    startTalking,
-    stopTalking,
-    activeSpeakerUserId,
-    isAdmin,
-    talkOverEnabled,
-    pttButtonHeld,
-    enablePTTButton,
-    setMicMuteWrapper,
-    client,
-    onClientSync,
-  ]);
-
   useEffect(() => {
     client.on(ClientEvent.Sync, onClientSync);
 

src/sound/PTTClips.tsx

@@ -42,7 +42,7 @@ export const PTTClips: React.FC<Props> = ({
   return (
     <>
       <audio
-        preload="true"
+        preload="auto"
         className={styles.pttClip}
         ref={startTalkingLocalRef}
       >
@@ -50,18 +50,18 @@ export const PTTClips: React.FC<Props> = ({
         <source type="audio/mpeg" src={startTalkLocalMp3Url} />
       </audio>
       <audio
-        preload="true"
+        preload="auto"
         className={styles.pttClip}
         ref={startTalkingRemoteRef}
       >
         <source type="audio/ogg" src={startTalkRemoteOggUrl} />
         <source type="audio/mpeg" src={startTalkRemoteMp3Url} />
       </audio>
-      <audio preload="true" className={styles.pttClip} ref={endTalkingRef}>
+      <audio preload="auto" className={styles.pttClip} ref={endTalkingRef}>
         <source type="audio/ogg" src={endTalkOggUrl} />
         <source type="audio/mpeg" src={endTalkMp3Url} />
       </audio>
-      <audio preload="true" className={styles.pttClip} ref={blockedRef}>
+      <audio preload="auto" className={styles.pttClip} ref={blockedRef}>
         <source type="audio/ogg" src={blockedOggUrl} />
         <source type="audio/mpeg" src={blockedMp3Url} />
       </audio>

src/useDelayedState.ts

@@ -0,0 +1,49 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { useState, useRef, useCallback } from "react";
+
+// Like useState, except state updates can be enqueued with a configurable delay
+export const useDelayedState = <T>(
+  initial?: T
+): [T, (value: T, delay: number) => void, (value: T) => void] => {
+  const [state, setState] = useState<T>(initial);
+  const timers = useRef<Set<ReturnType<typeof setTimeout>>>();
+  if (!timers.current) timers.current = new Set();
+
+  const setStateDelayed = useCallback(
+    (value: T, delay: number) => {
+      const timer = setTimeout(() => {
+        setState(value);
+        timers.current.delete(timer);
+      }, delay);
+      timers.current.add(timer);
+    },
+    [setState, timers]
+  );
+  const setStateImmediate = useCallback(
+    (value: T) => {
+      // Clear all updates currently in the queue
+      for (const timer of timers.current) clearTimeout(timer);
+      timers.current.clear();
+
+      setState(value);
+    },
+    [setState, timers]
+  );
+
+  return [state, setStateDelayed, setStateImmediate];
+};

src/useEvents.ts

@@ -0,0 +1,34 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { useEffect } from "react";
+
+// Shortcut for registering a listener on an EventTarget
+export const useEventTarget = <T extends Event>(
+  target: EventTarget,
+  eventType: string,
+  listener: (event: T) => void,
+  options?: AddEventListenerOptions
+) => {
+  useEffect(() => {
+    if (target) {
+      target.addEventListener(eventType, listener, options);
+      return () => target.removeEventListener(eventType, listener, options);
+    }
+  }, [target, eventType, listener, options]);
+};
+
+// TODO: Have a similar hook for EventEmitters