Merge pull request #24 from vector-im/feature/active-speaker-layout

Active Speaker Layout

public/browser-matrix.js

@@ -44766,7 +44766,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
+exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.CallFeed = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
 const memory_crypto_store_1 = require("./crypto/store/memory-crypto-store");
 const memory_1 = require("./store/memory");
 const scheduler_1 = require("./scheduler");
@@ -44800,6 +44800,11 @@ var call_1 = require("./webrtc/call");
 Object.defineProperty(exports, "createNewMatrixCall", { enumerable: true, get: function () { return call_1.createNewMatrixCall; } });
 Object.defineProperty(exports, "setMatrixCallAudioInput", { enumerable: true, get: function () { return call_1.setAudioInput; } });
 Object.defineProperty(exports, "setMatrixCallVideoInput", { enumerable: true, get: function () { return call_1.setVideoInput; } });
+// TODO: This export is temporary and is only used for the local call feed for conference calls
+// Ideally conference calls will become a first-class concept and we will have a local call feed with
+// a lifecycle that matches the conference call, not individual calls to members.
+var callFeed_1 = require("./webrtc/callFeed");
+Object.defineProperty(exports, "CallFeed", { enumerable: true, get: function () { return callFeed_1.CallFeed; } });
 // expose the underlying request object so different environments can use
 // different request libs (e.g. request or browser-request)
 let requestInstance;
@@ -44922,7 +44927,7 @@ exports.createClient = createClient;
 
 }).call(this)}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {})
 
-},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151}],120:[function(require,module,exports){
+},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151,"./webrtc/callFeed":154}],120:[function(require,module,exports){
 "use strict";
 /*
 Copyright 2021 The Matrix.org Foundation C.I.C.
@@ -59157,6 +59162,9 @@ class MatrixCall extends events_1.EventEmitter {
         logger_1.logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`);
     }
     deleteAllFeeds() {
+        for (const feed of this.feeds) {
+            feed.dispose();
+        }
         this.feeds = [];
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -59167,6 +59175,7 @@ class MatrixCall extends events_1.EventEmitter {
             logger_1.logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`);
             return;
         }
+        feed.dispose();
         this.feeds.splice(this.feeds.indexOf(feed), 1);
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -60737,7 +60746,7 @@ class CallFeed extends events_1.default {
     volumeLooper() {
         if (!this.analyser)
             return;
-        setTimeout(() => {
+        this.volumeLooperTimeout = setTimeout(() => {
             if (!this.measuringVolumeActivity)
                 return;
             this.analyser.getFloatFrequencyData(this.frequencyBinCount);
@@ -60756,6 +60765,9 @@ class CallFeed extends events_1.default {
             this.volumeLooper();
         }, POLLING_INTERVAL);
     }
+    dispose() {
+        clearTimeout(this.volumeLooperTimeout);
+    }
 }
 exports.CallFeed = CallFeed;
 

src/ConferenceCallManager.js

@@ -22,6 +22,8 @@ const CONF_ROOM = "me.robertlong.conf";
 const CONF_PARTICIPANT = "me.robertlong.conf.participant";
 const PARTICIPANT_TIMEOUT = 1000 * 15;
 const SPEAKING_THRESHOLD = -80;
+const ACTIVE_SPEAKER_INTERVAL = 1000;
+const ACTIVE_SPEAKER_SAMPLES = 8;
 
 function waitForSync(client) {
   return new Promise((resolve, reject) => {
@@ -172,10 +174,15 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.localVideoStream = null;
     this.localParticipant = null;
+    this.localCallFeed = null;
 
     this.audioMuted = false;
     this.videoMuted = false;
 
+    this.activeSpeaker = null;
+    this._speakerMap = new Map();
+    this._activeSpeakerLoopTimeout = null;
+
     this.client.on("RoomState.members", this._onRoomStateMembers);
     this.client.on("Call.incoming", this._onIncomingCall);
     this.callDebugger = new ConferenceCallDebugger(this);
@@ -249,11 +256,38 @@ export class ConferenceCallManager extends EventEmitter {
       audioMuted: this.audioMuted,
       videoMuted: this.videoMuted,
       speaking: false,
+      activeSpeaker: true,
     };
 
+    this.activeSpeaker = this.localParticipant;
+
     this.participants.push(this.localParticipant);
     this.emit("debugstate", userId, null, "you");
 
+    this.localCallFeed = new matrixcs.CallFeed(
+      stream,
+      this.localParticipant.userId,
+      "m.usermedia",
+      this.client,
+      this.room.roomId,
+      this.audioMuted,
+      this.videoMuted
+    );
+    this.localCallFeed.on("mute_state_changed", () =>
+      this._onCallFeedMuteStateChanged(
+        this.localParticipant,
+        this.localCallFeed
+      )
+    );
+    this.localCallFeed.setSpeakingThreshold(SPEAKING_THRESHOLD);
+    this.localCallFeed.measureVolumeActivity(true);
+    this.localCallFeed.on("speaking", (speaking) => {
+      this._onCallFeedSpeaking(this.localParticipant, speaking);
+    });
+    this.localCallFeed.on("volume_changed", (maxVolume) =>
+      this._onCallFeedVolumeChange(this.localParticipant, maxVolume)
+    );
+
     // Announce to the other room members that we have entered the room.
     // Continue doing so every PARTICIPANT_TIMEOUT ms
     this._updateMemberParticipantState();
@@ -278,6 +312,7 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.emit("entered");
     this.emit("participants_changed");
+    this._onActiveSpeakerLoop();
   }
 
   leaveCall() {
@@ -309,19 +344,20 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.client.stopLocalMediaStream();
     this.localVideoStream = null;
+    this.localCallFeed.dispose();
+    this.localCallFeed = null;
 
     this.room = null;
     this.entered = false;
     this._left = true;
     this.participants = [];
-    this.localParticipant.stream = null;
+    this.localParticipant = null;
-    this.localParticipant.call = null;
+    this.activeSpeaker = null;
-    this.localParticipant.audioMuted = false;
-    this.localParticipant.videoMuted = false;
-    this.localParticipant.speaking = false;
     this.audioMuted = false;
     this.videoMuted = false;
     clearTimeout(this._memberParticipantStateTimeout);
+    clearTimeout(this._activeSpeakerLoopTimeout);
+    this._speakerMap.clear();
 
     this.emit("participants_changed");
     this.emit("left");
@@ -342,8 +378,8 @@ export class ConferenceCallManager extends EventEmitter {
   setAudioMuted(muted) {
     this.audioMuted = muted;
 
-    if (this.localParticipant) {
+    if (this.localCallFeed) {
-      this.localParticipant.audioMuted = muted;
+      this.localCallFeed.setAudioMuted(muted);
     }
 
     const localStream = this.localVideoStream;
@@ -374,8 +410,8 @@ export class ConferenceCallManager extends EventEmitter {
   setVideoMuted(muted) {
     this.videoMuted = muted;
 
-    if (this.localParticipant) {
+    if (this.localCallFeed) {
-      this.localParticipant.videoMuted = muted;
+      this.localCallFeed.setVideoMuted(muted);
     }
 
     const localStream = this.localVideoStream;
@@ -506,6 +542,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
     }
 
     const userId = call.opponentMember.userId;
@@ -532,6 +571,7 @@ export class ConferenceCallManager extends EventEmitter {
       existingParticipant.audioMuted = audioMuted;
       existingParticipant.videoMuted = videoMuted;
       existingParticipant.speaking = false;
+      existingParticipant.activeSpeaker = false;
       existingParticipant.sessionId = sessionId;
     } else {
       participant = {
@@ -543,6 +583,7 @@ export class ConferenceCallManager extends EventEmitter {
         audioMuted,
         videoMuted,
         speaking: false,
+        activeSpeaker: false,
       };
       this.participants.push(participant);
     }
@@ -634,6 +675,7 @@ export class ConferenceCallManager extends EventEmitter {
       participant.audioMuted = false;
       participant.videoMuted = false;
       participant.speaking = false;
+      participant.activeSpeaker = false;
     } else {
       participant = {
         local: false,
@@ -644,6 +686,7 @@ export class ConferenceCallManager extends EventEmitter {
         audioMuted: false,
         videoMuted: false,
         speaking: false,
+        activeSpeaker: false,
       };
       // TODO: Should we wait until the call has been answered to push the participant?
       // Or do we hide the participant until their stream is live?
@@ -708,6 +751,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
       this._onCallFeedMuteStateChanged(participant, remoteFeed);
     }
   };
@@ -715,6 +761,14 @@ export class ConferenceCallManager extends EventEmitter {
   _onCallFeedMuteStateChanged = (participant, feed) => {
     participant.audioMuted = feed.isAudioMuted();
     participant.videoMuted = feed.isVideoMuted();
+
+    if (participant.audioMuted) {
+      this._speakerMap.set(
+        participant.userId,
+        Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
+      );
+    }
+
     this.emit("participants_changed");
   };
 
@@ -723,6 +777,61 @@ export class ConferenceCallManager extends EventEmitter {
     this.emit("participants_changed");
   };
 
+  _onCallFeedVolumeChange = (participant, maxVolume) => {
+    if (!this._speakerMap.has(participant.userId)) {
+      this._speakerMap.set(
+        participant.userId,
+        Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
+      );
+    }
+
+    const volumeArr = this._speakerMap.get(participant.userId);
+    volumeArr.shift();
+    volumeArr.push(maxVolume);
+  };
+
+  _onActiveSpeakerLoop = () => {
+    let topAvg;
+    let activeSpeakerId;
+
+    for (const [userId, volumeArr] of this._speakerMap) {
+      let total = 0;
+
+      for (let i = 0; i < volumeArr.length; i++) {
+        const volume = volumeArr[i];
+        total += Math.max(volume, SPEAKING_THRESHOLD);
+      }
+
+      const avg = total / ACTIVE_SPEAKER_SAMPLES;
+
+      if (!topAvg) {
+        topAvg = avg;
+        activeSpeakerId = userId;
+      } else if (avg > topAvg) {
+        topAvg = avg;
+        activeSpeakerId = userId;
+      }
+    }
+
+    if (activeSpeakerId && topAvg > SPEAKING_THRESHOLD) {
+      const nextActiveSpeaker = this.participants.find(
+        (p) => p.userId === activeSpeakerId
+      );
+
+      if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) {
+        this.activeSpeaker.activeSpeaker = false;
+        nextActiveSpeaker.activeSpeaker = true;
+        this.activeSpeaker = nextActiveSpeaker;
+        this.emit("participants_changed");
+      }
+    }
+
+    this._activeSpeakerLoopTimeout = setTimeout(
+      this._onActiveSpeakerLoop,
+      ACTIVE_SPEAKER_INTERVAL
+    );
+  };
+
   _onCallReplaced = (participant, call, newCall) => {
     participant.call = newCall;
 
@@ -751,6 +860,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
     }
 
     this.emit("call", newCall);
@@ -770,6 +882,13 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.participants.splice(participantIndex, 1);
 
+    if (this.activeSpeaker === participant && this.participants.length > 0) {
+      this.activeSpeaker = this.participants[0];
+      this.activeSpeaker.activeSpeaker = true;
+    }
+
+    this._speakerMap.delete(participant.userId);
+
     this.emit("participants_changed");
   };
 

src/Room.jsx

@@ -14,7 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useEffect, useMemo, useState, useRef } from "react";
+import React, {
+  useEffect,
+  useMemo,
+  useState,
+  useRef,
+  useCallback,
+} from "react";
 import styles from "./Room.module.css";
 import { useParams, useLocation } from "react-router-dom";
 import { useVideoRoom } from "./ConferenceCallManagerHooks";
@@ -25,6 +31,7 @@ import {
   SettingsButton,
   MicButton,
   VideoButton,
+  LayoutToggleButton,
 } from "./RoomButton";
 import { Header, LeftNav, RightNav, CenterNav } from "./Header";
 import { Button } from "./Input";
@@ -71,6 +78,12 @@ export function Room({ manager }) {
     };
   }, []);
 
+  const [layout, setLayout] = useState("gallery");
+
+  const toggleLayout = useCallback(() => {
+    setLayout(layout === "spotlight" ? "gallery" : "spotlight");
+  }, [layout]);
+
   return (
     <div className={styles.room}>
       {!loading && room && (
@@ -80,6 +93,13 @@ export function Room({ manager }) {
             <h3>{room.name}</h3>
           </CenterNav>
           <RightNav>
+            {!loading && room && joined && (
+              <LayoutToggleButton
+                title={layout === "spotlight" ? "Spotlight" : "Gallery"}
+                layout={layout}
+                onClick={toggleLayout}
+              />
+            )}
             <SettingsButton
               title={debug ? "Disable DevTools" : "Enable DevTools"}
               on={debug}
@@ -111,7 +131,7 @@ export function Room({ manager }) {
         </div>
       )}
       {!loading && room && joined && participants.length > 0 && (
-        <VideoGrid participants={participants} />
+        <VideoGrid participants={participants} layout={layout} />
       )}
       {!loading && room && joined && (
         <div className={styles.footer}>

src/RoomButton.jsx

@@ -7,6 +7,8 @@ import { ReactComponent as VideoIcon } from "./icons/Video.svg";
 import { ReactComponent as DisableVideoIcon } from "./icons/DisableVideo.svg";
 import { ReactComponent as HangupIcon } from "./icons/Hangup.svg";
 import { ReactComponent as SettingsIcon } from "./icons/Settings.svg";
+import { ReactComponent as GridIcon } from "./icons/Grid.svg";
+import { ReactComponent as SpeakerIcon } from "./icons/Speaker.svg";
 
 export function RoomButton({ on, className, children, ...rest }) {
   return (
@@ -66,3 +68,15 @@ export function SettingsButton(props) {
     </HeaderButton>
   );
 }
+
+export function LayoutToggleButton({ layout, ...rest }) {
+  return (
+    <HeaderButton {...rest}>
+      {layout === "spotlight" ? (
+        <SpeakerIcon width={20} height={20} />
+      ) : (
+        <GridIcon width={20} height={20} />
+      )}
+    </HeaderButton>
+  );
+}

src/RoomButton.module.css

@@ -49,6 +49,10 @@ limitations under the License.
   border-radius: 32px;
 }
 
+.headerButton svg * {
+  fill: #8E99A4;
+}
+
 .headerButton:hover {
   background-color: #8D97A5;
 }

src/VideoGrid.jsx

@@ -403,13 +403,14 @@ function getSubGridPositions(tileCount, gridWidth, gridHeight, gap) {
   return newTilePositions;
 }
 
-export function VideoGrid({ participants }) {
+export function VideoGrid({ participants, layout }) {
   const [{ tiles, tilePositions }, setTileState] = useState({
     tiles: [],
     tilePositions: [],
   });
   const draggingTileRef = useRef(null);
   const lastTappedRef = useRef({});
+  const lastLayoutRef = useRef(layout);
   const isMounted = useIsMounted();
 
   const [gridRef, gridBounds] = useMeasure();
@@ -418,35 +419,34 @@ export function VideoGrid({ participants }) {
     setTileState(({ tiles }) => {
       const newTiles = [];
       const removedTileKeys = [];
-      let presenterTileCount = 0;
 
       for (const tile of tiles) {
-        const participant = participants.find(
+        let participant = participants.find(
           (participant) => participant.userId === tile.key
         );
 
-        if (tile.presenter) {
+        let remove = false;
-          presenterTileCount++;
+
+        if (!participant) {
+          remove = true;
+          participant = tile.participant;
+          removedTileKeys.push(tile.key);
         }
 
-        if (participant) {
+        let presenter;
-          // Existing tiles
+
-          newTiles.push({
+        if (layout === "spotlight") {
-            key: participant.userId,
+          presenter = participant.activeSpeaker;
-            participant: participant,
-            remove: false,
-            presenter: tile.presenter,
-          });
         } else {
-          // Removed tiles
+          presenter = layout === lastLayoutRef.current ? tile.presenter : false;
-          removedTileKeys.push(tile.key);
-          newTiles.push({
-            key: tile.key,
-            participant: tile.participant,
-            remove: true,
-            presenter: tile.presenter,
-          });
         }
+
+        newTiles.push({
+          key: participant.userId,
+          participant,
+          remove,
+          presenter,
+        });
       }
 
       for (const participant of participants) {
@@ -459,7 +459,7 @@ export function VideoGrid({ participants }) {
           key: participant.userId,
           participant,
           remove: false,
-          presenter: false,
+          presenter: layout === "spotlight" && participant.activeSpeaker,
         });
       }
 
@@ -476,6 +476,11 @@ export function VideoGrid({ participants }) {
               (tile) => !removedTileKeys.includes(tile.key)
             );
 
+            const presenterTileCount = newTiles.reduce(
+              (count, tile) => count + (tile.presenter ? 1 : 0),
+              0
+            );
+
             return {
               tiles: newTiles,
               tilePositions: getTilePositions(
@@ -489,6 +494,13 @@ export function VideoGrid({ participants }) {
         }, 250);
       }
 
+      const presenterTileCount = newTiles.reduce(
+        (count, tile) => count + (tile.presenter ? 1 : 0),
+        0
+      );
+
+      lastLayoutRef.current = layout;
+
       return {
         tiles: newTiles,
         tilePositions: getTilePositions(
@@ -499,7 +511,7 @@ export function VideoGrid({ participants }) {
         ),
       };
     });
-  }, [participants, gridBounds]);
+  }, [participants, gridBounds, layout]);
 
   const animate = useCallback(
     (tiles) => (tileIndex) => {
@@ -673,7 +685,7 @@ export function VideoGrid({ participants }) {
 
       api.start(animate(newTiles));
     },
-    { filterTaps: true }
+    { filterTaps: true, enabled: layout === "gallery" }
   );
 
   return (
@@ -699,6 +711,10 @@ export function VideoGrid({ participants }) {
   );
 }
 
+VideoGrid.defaultProps = {
+  layout: "gallery",
+};
+
 function ParticipantTile({ style, participant, remove, presenter, ...rest }) {
   const videoRef = useRef();
 

src/icons/Grid.svg

@@ -1,11 +1,11 @@
 <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="1" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="1" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="1" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="1" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="1" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="1" width="6" height="6" rx="1" fill="white"/>
 </svg>

src/icons/Speaker.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="18" y="10" width="5" height="5" rx="1" fill="white"/>
+<rect x="18" y="16" width="5" height="5" rx="1" fill="white"/>
+<rect x="18" y="4" width="5" height="5" rx="1" fill="white"/>
+<rect x="1" y="4" width="16" height="17" rx="1" fill="white"/>
+</svg>