Merge remote-tracking branch 'upstream/main' into SimonBrandner/feat/audio-share

package.json

@@ -38,7 +38,7 @@
     "classnames": "^2.3.1",
     "color-hash": "^2.0.1",
     "events": "^3.3.0",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#984dd26a138411ef73903ff4e635f2752e0829f2",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e876482e62421bb6a1ba58078435c6b3d1210f15",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",
     "pako": "^2.0.4",

src/App.jsx

@@ -26,15 +26,21 @@ import { RoomRedirect } from "./room/RoomRedirect";
 import { ClientProvider } from "./ClientContext";
 import { usePageFocusStyle } from "./usePageFocusStyle";
 import { SequenceDiagramViewerPage } from "./SequenceDiagramViewerPage";
+import { InspectorContextProvider } from "./room/GroupCallInspector";
+import { CrashView } from "./FullScreenView";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
 
 export default function App({ history }) {
   usePageFocusStyle();
 
+  const errorPage = <CrashView />;
+
   return (
     <Router history={history}>
       <ClientProvider>
+        <InspectorContextProvider>
+          <Sentry.ErrorBoundary fallback={errorPage}>
             <OverlayProvider>
               <Switch>
                 <SentryRoute exact path="/">
@@ -57,6 +63,8 @@ export default function App({ history }) {
                 </SentryRoute>
               </Switch>
             </OverlayProvider>
+          </Sentry.ErrorBoundary>
+        </InspectorContextProvider>
       </ClientProvider>
     </Router>
   );

src/FullScreenView.jsx

@@ -4,6 +4,8 @@ import styles from "./FullScreenView.module.css";
 import { Header, HeaderLogo, LeftNav, RightNav } from "./Header";
 import classNames from "classnames";
 import { LinkButton, Button } from "./button";
+import { useSubmitRageshake } from "./settings/submit-rageshake";
+import { ErrorMessage } from "./input/Input";
 
 export function FullScreenView({ className, children }) {
   return (
@@ -59,6 +61,56 @@ export function ErrorView({ error }) {
   );
 }
 
+export function CrashView() {
+  const { submitRageshake, sending, sent, error } = useSubmitRageshake();
+
+  const sendDebugLogs = useCallback(() => {
+    submitRageshake({
+      description: "**Soft Crash**",
+      sendLogs: true,
+    });
+  }, [submitRageshake]);
+
+  const onReload = useCallback(() => {
+    window.location = "/";
+  }, []);
+
+  let logsComponent;
+  if (sent) {
+    logsComponent = <div>Thanks! We'll get right on it.</div>;
+  } else if (sending) {
+    logsComponent = <div>Sending...</div>;
+  } else {
+    logsComponent = (
+      <Button
+        size="lg"
+        variant="default"
+        onPress={sendDebugLogs}
+        className={styles.wideButton}
+      >
+        Send debug logs
+      </Button>
+    );
+  }
+
+  return (
+    <FullScreenView>
+      <h1>Oops, something's gone wrong.</h1>
+      <p>Submitting debug logs will help us track down the problem.</p>
+      <div className={styles.sendLogsSection}>{logsComponent}</div>
+      {error && <ErrorMessage>Couldn't send debug logs!</ErrorMessage>}
+      <Button
+        size="lg"
+        variant="default"
+        className={styles.wideButton}
+        onPress={onReload}
+      >
+        Return to home screen
+      </Button>
+    </FullScreenView>
+  );
+}
+
 export function LoadingView() {
   return (
     <FullScreenView>

src/FullScreenView.module.css

@@ -36,6 +36,12 @@
   margin-bottom: 0;
 }
 
-.homeLink {
+/* Make the buttons the same width */
+.wideButton {
   width: 291px;
 }
+
+/* Fixed height to avoid content jumping around*/
+.sendLogsSection {
+  height: 50px;
+}

src/button/Button.tsx

@@ -30,6 +30,7 @@ import { ReactComponent as SettingsIcon } from "../icons/Settings.svg";
 import { ReactComponent as AddUserIcon } from "../icons/AddUser.svg";
 import { ReactComponent as ArrowDownIcon } from "../icons/ArrowDown.svg";
 import { TooltipTrigger } from "../Tooltip";
+import { ReactComponent as OverflowIcon } from "../icons/Overflow.svg";
 
 export type ButtonVariant =
   | "default"
@@ -237,3 +238,14 @@ export function InviteButton({
     </TooltipTrigger>
   );
 }
+
+export function OptionsButton(props: Omit<Props, "variant">) {
+  return (
+    <TooltipTrigger>
+      <Button variant="icon" {...props}>
+        <OverflowIcon />
+      </Button>
+      {() => "Options"}
+    </TooltipTrigger>
+  );
+}

src/home/RegisteredView.jsx

@@ -47,7 +47,7 @@ export function RegisteredView({ client }) {
         setError(undefined);
         setLoading(true);
 
-        const roomIdOrAlias = await createRoom(client, roomName, ptt);
+        const [roomIdOrAlias] = await createRoom(client, roomName, ptt);
 
         if (roomIdOrAlias) {
           history.push(`/room/${roomIdOrAlias}`);

src/home/UnauthenticatedView.jsx

@@ -70,7 +70,7 @@ export function UnauthenticatedView() {
 
         let roomIdOrAlias;
         try {
-          roomIdOrAlias = await createRoom(client, roomName, ptt);
+          [roomIdOrAlias] = await createRoom(client, roomName, ptt);
         } catch (error) {
           if (error.errcode === "M_ROOM_IN_USE") {
             setOnFinished(() => () => {

src/main.tsx

@@ -28,9 +28,7 @@ import { Integrations } from "@sentry/tracing";
 
 import "./index.css";
 import App from "./App";
-import { ErrorView } from "./FullScreenView";
 import { init as initRageshake } from "./settings/rageshake";
-import { InspectorContextProvider } from "./room/GroupCallInspector";
 
 initRageshake();
 
@@ -104,11 +102,7 @@ Sentry.init({
 
 ReactDOM.render(
   <React.StrictMode>
-    <Sentry.ErrorBoundary fallback={ErrorView}>
-      <InspectorContextProvider>
     <App history={history} />
-      </InspectorContextProvider>
-    </Sentry.ErrorBoundary>
   </React.StrictMode>,
   document.getElementById("root")
 );

src/matrix-utils.ts

@@ -220,8 +220,8 @@ export function isLocalRoomId(roomId: string): boolean {
 export async function createRoom(
   client: MatrixClient,
   name: string
-): Promise<string> {
+): Promise<[string, string]> {
-  await client.createRoom({
+  const result = await client.createRoom({
     visibility: Visibility.Private,
     preset: Preset.PublicChat,
     name,
@@ -251,7 +251,7 @@ export async function createRoom(
     },
   });
 
-  return fullAliasFromRoomName(name, client);
+  return [fullAliasFromRoomName(name, client), result.room_id];
 }
 
 export function getRoomUrl(roomId: string): string {

src/room/useLoadGroupCall.ts

@@ -21,6 +21,7 @@ import {
   GroupCallIntent,
 } from "matrix-js-sdk/src/webrtc/groupCall";
 import { GroupCallEventHandlerEvent } from "matrix-js-sdk/src/webrtc/groupCallEventHandler";
+import { ClientEvent } from "matrix-js-sdk/src/client";
 
 import type { MatrixClient } from "matrix-js-sdk/src/client";
 import type { Room } from "matrix-js-sdk/src/models/room";
@@ -44,9 +45,38 @@ export const useLoadGroupCall = (
   useEffect(() => {
     setState({ loading: true });
 
+    const waitForRoom = async (roomId: string): Promise<Room> => {
+      const room = client.getRoom(roomId);
+      if (room) return room;
+      console.log(`Room ${roomId} hasn't arrived yet: waiting`);
+
+      const waitPromise = new Promise<Room>((resolve) => {
+        const onRoomEvent = async (room: Room) => {
+          if (room.roomId === roomId) {
+            client.removeListener(ClientEvent.Room, onRoomEvent);
+            resolve(room);
+          }
+        };
+        client.on(ClientEvent.Room, onRoomEvent);
+      });
+
+      // race the promise with a timeout so we don't
+      // wait forever for the room
+      const timeoutPromise = new Promise<Room>((_, reject) => {
+        setTimeout(() => {
+          reject(new Error("Timed out trying to join room"));
+        }, 30000);
+      });
+
+      return Promise.race([waitPromise, timeoutPromise]);
+    };
+
     const fetchOrCreateRoom = async (): Promise<Room> => {
       try {
-        return await client.joinRoom(roomIdOrAlias, { viaServers });
+        const room = await client.joinRoom(roomIdOrAlias, { viaServers });
+        // wait for the room to come down the sync stream, otherwise
+        // client.getRoom() won't return the room.
+        return waitForRoom(room.roomId);
       } catch (error) {
         if (
           isLocalRoomId(roomIdOrAlias) &&
@@ -55,8 +85,12 @@ export const useLoadGroupCall = (
               error.message.indexOf("Failed to fetch alias") !== -1))
         ) {
           // The room doesn't exist, but we can create it
-          await createRoom(client, roomNameFromRoomId(roomIdOrAlias));
+          const [, roomId] = await createRoom(
-          return await client.joinRoom(roomIdOrAlias, { viaServers });
+            client,
+            roomNameFromRoomId(roomIdOrAlias)
+          );
+          // likewise, wait for the room
+          return await waitForRoom(roomId);
         } else {
           throw error;
         }

src/video-grid/VideoTile.jsx

@@ -20,6 +20,7 @@ import classNames from "classnames";
 import styles from "./VideoTile.module.css";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
+import { OptionsButton } from "../button/Button";
 
 export const VideoTile = forwardRef(
   (
@@ -35,6 +36,8 @@ export const VideoTile = forwardRef(
       name,
       showName,
       mediaRef,
+      onOptionsPress,
+      showOptions,
       ...rest
     },
     ref
@@ -62,13 +65,18 @@ export const VideoTile = forwardRef(
           </div>
         ) : (
           (showName || audioMuted || (videoMuted && !noVideo)) && (
-            <div className={styles.memberName}>
+            <div className={classNames(styles.infoBubble, styles.memberName)}>
               {audioMuted && !(videoMuted && !noVideo) && <MicMutedIcon />}
               {videoMuted && !noVideo && <VideoMutedIcon />}
               {showName && <span title={name}>{name}</span>}
             </div>
           )
         )}
+        {showOptions && (
+          <div className={classNames(styles.infoBubble, styles.optionsButton)}>
+            <OptionsButton onPress={onOptionsPress} />
+          </div>
+        )}
         <video ref={mediaRef} playsInline disablePictureInPicture />
       </animated.div>
     );

src/video-grid/VideoTile.module.css

@@ -44,10 +44,8 @@
   object-fit: contain;
 }
 
-.memberName {
+.infoBubble {
   position: absolute;
-  bottom: 16px;
-  left: 16px;
   height: 24px;
   padding: 0 8px;
   color: white;
@@ -62,6 +60,21 @@
   z-index: 1;
 }
 
+.optionsButton {
+  right: 16px;
+  top: 16px;
+}
+
+.optionsButton button svg {
+  height: 20px;
+  width: 20px;
+}
+
+.memberName {
+  left: 16px;
+  bottom: 16px;
+}
+
 .memberName > * {
   margin-right: 6px;
 }

src/video-grid/VideoTileContainer.jsx

@@ -20,6 +20,8 @@ import { useCallFeed } from "./useCallFeed";
 import { useSpatialMediaStream } from "./useMediaStream";
 import { useRoomMemberName } from "./useRoomMemberName";
 import { VideoTile } from "./VideoTile";
+import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
+import { useModalTriggerState } from "../Modal";
 
 export function VideoTileContainer({
   item,
@@ -37,6 +39,7 @@ export function VideoTileContainer({
     isLocal,
     audioMuted,
     videoMuted,
+    localVolume,
     noVideo,
     speaking,
     stream,
@@ -49,13 +52,22 @@ export function VideoTileContainer({
     audioOutputDevice,
     audioContext,
     audioDestination,
-    isLocal
+    isLocal,
+    localVolume
   );
+  const {
+    modalState: videoTileSettingsModalState,
+    modalProps: videoTileSettingsModalProps,
+  } = useModalTriggerState();
+  const onOptionsPress = () => {
+    videoTileSettingsModalState.open();
+  };
 
   // Firefox doesn't respect the disablePictureInPicture attribute
   // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
 
   return (
+    <>
       <VideoTile
         isLocal={isLocal}
         speaking={speaking && !disableSpeakingIndicator}
@@ -68,7 +80,16 @@ export function VideoTileContainer({
         ref={tileRef}
         mediaRef={mediaRef}
         avatar={getAvatar && getAvatar(member, width, height)}
+        onOptionsPress={onOptionsPress}
+        showOptions={!item.callFeed.isLocal()}
         {...rest}
       />
+      {videoTileSettingsModalState.isOpen && (
+        <VideoTileSettingsModal
+          {...videoTileSettingsModalProps}
+          feed={item.callFeed}
+        />
+      )}
+    </>
   );
 }

src/video-grid/VideoTileSettingsModal.module.css

@@ -0,0 +1,70 @@
+.content {
+  margin: 27px 34px;
+}
+
+.localVolumePercentage {
+  width: 3ch;
+}
+
+.localVolumeSlider[type="range"] {
+  -ms-appearance: none;
+  -moz-appearance: none;
+  -webkit-appearance: none;
+  appearance: none;
+
+  cursor: pointer;
+  width: 100%;
+}
+
+.localVolumeSlider[type="range"]::-moz-range-track {
+  -moz-appearance: none;
+  appearance: none;
+
+  height: 4px;
+}
+.localVolumeSlider[type="range"]::-ms-track {
+  -ms-appearance: none;
+  appearance: none;
+
+  height: 4px;
+}
+.localVolumeSlider[type="range"]::-webkit-slider-runnable-track {
+  -webkit-appearance: none;
+  appearance: none;
+
+  height: 4px;
+}
+
+.localVolumeSlider[type="range"]::-moz-range-thumb {
+  -moz-appearance: none;
+  appearance: none;
+
+  height: 16px;
+  width: 16px;
+  margin-top: -6px;
+
+  border-radius: 100%;
+  background: var(--accent);
+}
+.localVolumeSlider[type="range"]::-ms-thumb {
+  -ms-appearance: none;
+  appearance: none;
+
+  height: 16px;
+  width: 16px;
+  margin-top: -6px;
+
+  border-radius: 100%;
+  background: var(--accent);
+}
+.localVolumeSlider[type="range"]::-webkit-slider-thumb {
+  -webkit-appearance: none;
+  appearance: none;
+
+  height: 16px;
+  width: 16px;
+  margin-top: -6px;
+
+  border-radius: 100%;
+  background: var(--accent);
+}

src/video-grid/VideoTileSettingsModal.tsx

@@ -0,0 +1,74 @@
+/*
+Copyright 2022 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import React, { ChangeEvent, useState } from "react";
+import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
+
+import selectInputStyles from "../input/SelectInput.module.css";
+import { FieldRow } from "../input/Input";
+import { Modal } from "../Modal";
+import styles from "./VideoTileSettingsModal.module.css";
+
+interface LocalVolumeProps {
+  feed: CallFeed;
+}
+
+const LocalVolume: React.FC<LocalVolumeProps> = ({
+  feed,
+}: LocalVolumeProps) => {
+  const [localVolume, setLocalVolume] = useState<number>(feed.getLocalVolume());
+
+  const onLocalVolumeChanged = (event: ChangeEvent<HTMLInputElement>) => {
+    const value: number = +event.target.value;
+    setLocalVolume(value);
+    feed.setLocalVolume(value);
+  };
+
+  return (
+    <>
+      <h4 className={selectInputStyles.label}> Local Volume </h4>
+      <FieldRow>
+        <span className={styles.localVolumePercentage}>
+          {`${Math.round(localVolume * 100)}%`}
+        </span>
+        <input
+          className={styles.localVolumeSlider}
+          type="range"
+          min="0"
+          max="1"
+          step="0.01"
+          value={localVolume}
+          onChange={onLocalVolumeChanged}
+        />
+      </FieldRow>
+    </>
+  );
+};
+
+// TODO: Extend ModalProps
+interface Props {
+  feed: CallFeed;
+}
+
+export const VideoTileSettingsModal = ({ feed, ...rest }: Props) => {
+  return (
+    <Modal title="Feed settings" isDismissable mobileFullScreen {...rest}>
+      <div className={styles.content}>
+        <LocalVolume feed={feed} />
+      </div>
+    </Modal>
+  );
+};

src/video-grid/useCallFeed.js

@@ -27,6 +27,7 @@ function getCallFeedState(callFeed) {
       : true,
     videoMuted: callFeed ? callFeed.isVideoMuted() : true,
     audioMuted: callFeed ? callFeed.isAudioMuted() : true,
+    localVolume: callFeed ? callFeed.getLocalVolume() : 0,
     stream: callFeed ? callFeed.stream : undefined,
     purpose: callFeed ? callFeed.purpose : undefined,
   };
@@ -44,6 +45,10 @@ export function useCallFeed(callFeed) {
       setState((prevState) => ({ ...prevState, audioMuted, videoMuted }));
     }
 
+    function onLocalVolumeChanged(localVolume) {
+      setState((prevState) => ({ ...prevState, localVolume }));
+    }
+
     function onUpdateCallFeed() {
       setState(getCallFeedState(callFeed));
     }
@@ -51,6 +56,7 @@ export function useCallFeed(callFeed) {
     if (callFeed) {
       callFeed.on(CallFeedEvent.Speaking, onSpeaking);
       callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
+      callFeed.on(CallFeedEvent.LocalVolumeChanged, onLocalVolumeChanged);
       callFeed.on(CallFeedEvent.NewStream, onUpdateCallFeed);
     }
 
@@ -63,6 +69,10 @@ export function useCallFeed(callFeed) {
           CallFeedEvent.MuteStateChanged,
           onMuteStateChanged
         );
+        callFeed.removeListener(
+          CallFeedEvent.LocalVolumeChanged,
+          onLocalVolumeChanged
+        );
         callFeed.removeListener(CallFeedEvent.NewStream, onUpdateCallFeed);
       }
     };

src/video-grid/useMediaStream.ts

@@ -33,7 +33,8 @@ declare global {
 export const useMediaStream = (
   stream: MediaStream,
   audioOutputDevice: string,
-  mute = false
+  mute = false,
+  localVolume: number
 ): RefObject<MediaElement> => {
   const mediaRef = useRef<MediaElement>();
 
@@ -84,6 +85,13 @@ export const useMediaStream = (
     }
   }, [audioOutputDevice]);
 
+  useEffect(() => {
+    if (!mediaRef.current) return;
+    if (localVolume === null || localVolume === undefined) return;
+
+    mediaRef.current.volume = localVolume;
+  }, [localVolume]);
+
   useEffect(() => {
     const mediaEl = mediaRef.current;
 
@@ -187,7 +195,8 @@ export const useSpatialMediaStream = (
   audioOutputDevice: string,
   audioContext: AudioContext,
   audioDestination: AudioNode,
-  mute = false
+  mute = false,
+  localVolume: number
 ): [RefObject<Element>, RefObject<MediaElement>] => {
   const tileRef = useRef<Element>();
   const [spatialAudio] = useSpatialAudio();
@@ -195,26 +204,39 @@ export const useSpatialMediaStream = (
   const mediaRef = useMediaStream(
     stream,
     audioOutputDevice,
-    spatialAudio || mute
+    spatialAudio || mute,
+    localVolume
   );
 
+  const gainNodeRef = useRef<GainNode>();
   const pannerNodeRef = useRef<PannerNode>();
   const sourceRef = useRef<MediaStreamAudioSourceNode>();
 
   useEffect(() => {
-    if (spatialAudio && tileRef.current && !mute) {
+    if (
+      spatialAudio &&
+      tileRef.current &&
+      !mute &&
+      stream.getAudioTracks().length > 0
+    ) {
       if (!pannerNodeRef.current) {
         pannerNodeRef.current = new PannerNode(audioContext, {
           panningModel: "HRTF",
           refDistance: 3,
         });
       }
+      if (!gainNodeRef.current) {
+        gainNodeRef.current = new GainNode(audioContext, {
+          gain: localVolume,
+        });
+      }
       if (!sourceRef.current) {
         sourceRef.current = audioContext.createMediaStreamSource(stream);
       }
 
       const tile = tileRef.current;
       const source = sourceRef.current;
+      const gainNode = gainNodeRef.current;
       const pannerNode = pannerNodeRef.current;
 
       const updatePosition = () => {
@@ -229,8 +251,9 @@ export const useSpatialMediaStream = (
         pannerNodeRef.current.positionZ.value = -2;
       };
 
+      gainNode.gain.value = localVolume;
       updatePosition();
-      source.connect(pannerNode).connect(audioDestination);
+      source.connect(gainNode).connect(pannerNode).connect(audioDestination);
       // HACK: We abuse the CSS transitionrun event to detect when the tile
       // moves, because useMeasure, IntersectionObserver, etc. all have no
       // ability to track changes in the CSS transform property
@@ -239,10 +262,11 @@ export const useSpatialMediaStream = (
       return () => {
         tile.removeEventListener("transitionrun", updatePosition);
         source.disconnect();
+        gainNode.disconnect();
         pannerNode.disconnect();
       };
     }
-  }, [stream, spatialAudio, audioContext, audioDestination, mute]);
+  }, [stream, spatialAudio, audioContext, audioDestination, mute, localVolume]);
 
   return [tileRef, mediaRef];
 };

yarn.lock

@@ -8387,11 +8387,12 @@ matrix-events-sdk@^0.0.1-beta.7:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1-beta.7.tgz#5ffe45eba1f67cc8d7c2377736c728b322524934"
   integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#984dd26a138411ef73903ff4e635f2752e0829f2":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#e876482e62421bb6a1ba58078435c6b3d1210f15":
-  version "18.1.0"
+  version "19.0.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/984dd26a138411ef73903ff4e635f2752e0829f2"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/e876482e62421bb6a1ba58078435c6b3d1210f15"
   dependencies:
     "@babel/runtime" "^7.12.5"
+    "@types/sdp-transform" "^2.4.5"
     another-json "^0.2.0"
     browser-request "^0.3.3"
     bs58 "^4.0.1"
@@ -8401,6 +8402,7 @@ matrix-events-sdk@^0.0.1-beta.7:
     p-retry "^4.5.0"
     qs "^6.9.6"
     request "^2.88.2"
+    sdp-transform "^2.14.1"
     unhomoglyph "^1.0.6"
 
 md5.js@^1.3.4: