Active speaker tracking first draft

src/ConferenceCallManager.js

@@ -176,6 +176,10 @@ export class ConferenceCallManager extends EventEmitter {
     this.audioMuted = false;
     this.videoMuted = false;
 
+    this.activeSpeaker = null;
+    this._speakerMap = new Map();
+    this._activeSpeakerLoopTimeout = null;
+
     this.client.on("RoomState.members", this._onRoomStateMembers);
     this.client.on("Call.incoming", this._onIncomingCall);
     this.callDebugger = new ConferenceCallDebugger(this);
@@ -249,8 +253,11 @@ export class ConferenceCallManager extends EventEmitter {
       audioMuted: this.audioMuted,
       videoMuted: this.videoMuted,
       speaking: false,
+      activeSpeaker: true,
     };
 
+    this.activeSpeaker = this.localParticipant;
+
     this.participants.push(this.localParticipant);
     this.emit("debugstate", userId, null, "you");
 
@@ -278,6 +285,8 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.emit("entered");
     this.emit("participants_changed");
+    console.log("enter");
+    this._onActiveSpeakerLoop();
   }
 
   leaveCall() {
@@ -314,14 +323,12 @@ export class ConferenceCallManager extends EventEmitter {
     this.entered = false;
     this._left = true;
     this.participants = [];
-    this.localParticipant.stream = null;
+    this.localParticipant = null;
-    this.localParticipant.call = null;
+    this.activeSpeaker = null;
-    this.localParticipant.audioMuted = false;
-    this.localParticipant.videoMuted = false;
-    this.localParticipant.speaking = false;
     this.audioMuted = false;
     this.videoMuted = false;
     clearTimeout(this._memberParticipantStateTimeout);
+    clearTimeout(this._activeSpeakerLoopTimeout);
 
     this.emit("participants_changed");
     this.emit("left");
@@ -506,6 +513,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
     }
 
     const userId = call.opponentMember.userId;
@@ -532,6 +542,7 @@ export class ConferenceCallManager extends EventEmitter {
       existingParticipant.audioMuted = audioMuted;
       existingParticipant.videoMuted = videoMuted;
       existingParticipant.speaking = false;
+      existingParticipant.activeSpeaker = false;
       existingParticipant.sessionId = sessionId;
     } else {
       participant = {
@@ -543,6 +554,7 @@ export class ConferenceCallManager extends EventEmitter {
         audioMuted,
         videoMuted,
         speaking: false,
+        activeSpeaker: false,
       };
       this.participants.push(participant);
     }
@@ -634,6 +646,7 @@ export class ConferenceCallManager extends EventEmitter {
       participant.audioMuted = false;
       participant.videoMuted = false;
       participant.speaking = false;
+      participant.activeSpeaker = false;
     } else {
       participant = {
         local: false,
@@ -644,6 +657,7 @@ export class ConferenceCallManager extends EventEmitter {
         audioMuted: false,
         videoMuted: false,
         speaking: false,
+        activeSpeaker: false,
       };
       // TODO: Should we wait until the call has been answered to push the participant?
       // Or do we hide the participant until their stream is live?
@@ -708,6 +722,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
       this._onCallFeedMuteStateChanged(participant, remoteFeed);
     }
   };
@@ -715,6 +732,16 @@ export class ConferenceCallManager extends EventEmitter {
   _onCallFeedMuteStateChanged = (participant, feed) => {
     participant.audioMuted = feed.isAudioMuted();
     participant.videoMuted = feed.isVideoMuted();
+
+    if (participant.audioMuted) {
+      this._speakerMap.set(participant.userId, [
+        -Infinity,
+        -Infinity,
+        -Infinity,
+        -Infinity,
+      ]);
+    }
+
     this.emit("participants_changed");
   };
 
@@ -723,6 +750,61 @@ export class ConferenceCallManager extends EventEmitter {
     this.emit("participants_changed");
   };
 
+  _onCallFeedVolumeChange = (participant, maxVolume) => {
+    if (!this._speakerMap.has(participant.userId)) {
+      this._speakerMap.set(participant.userId, [
+        -Infinity,
+        -Infinity,
+        -Infinity,
+        -Infinity,
+      ]);
+    }
+
+    const volumeArr = this._speakerMap.get(participant.userId);
+    volumeArr.shift();
+    volumeArr.push(maxVolume);
+  };
+
+  _onActiveSpeakerLoop = () => {
+    let topAvg;
+    let activeSpeakerId;
+
+    for (const [userId, volumeArr] of this._speakerMap) {
+      let total = 0;
+
+      for (let i = 0; i < volumeArr.length; i++) {
+        const volume = volumeArr[i];
+        total += Math.max(volume, SPEAKING_THRESHOLD);
+      }
+
+      const avg = total / 4;
+
+      if (!topAvg) {
+        topAvg = avg;
+        activeSpeakerId = userId;
+      } else if (avg > topAvg) {
+        topAvg = avg;
+        activeSpeakerId = userId;
+      }
+    }
+
+    if (activeSpeakerId) {
+      const nextActiveSpeaker = this.participants.find(
+        (p) => p.userId === activeSpeakerId
+      );
+
+      if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) {
+        this.activeSpeaker.activeSpeaker = false;
+        nextActiveSpeaker.activeSpeaker = true;
+        this.activeSpeaker = nextActiveSpeaker;
+        console.log("activeSpeakerChanged", this.activeSpeaker);
+        this.emit("participants_changed");
+      }
+    }
+
+    this._activeSpeakerLoopTimeout = setTimeout(this._onActiveSpeakerLoop, 250);
+  };
+
   _onCallReplaced = (participant, call, newCall) => {
     participant.call = newCall;
 
@@ -751,6 +833,9 @@ export class ConferenceCallManager extends EventEmitter {
       remoteFeed.on("speaking", (speaking) => {
         this._onCallFeedSpeaking(participant, speaking);
       });
+      remoteFeed.on("volume_changed", (maxVolume) =>
+        this._onCallFeedVolumeChange(participant, maxVolume)
+      );
     }
 
     this.emit("call", newCall);
@@ -770,6 +855,13 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.participants.splice(participantIndex, 1);
 
+    if (this.activeSpeaker === participant && this.participants.length > 0) {
+      this.activeSpeaker = this.participants[0];
+      this.activeSpeaker.activeSpeaker = true;
+    }
+
+    this._speakerMap.delete(participant.userId);
+
     this.emit("participants_changed");
   };
 

src/Room.jsx

@@ -14,7 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { useEffect, useMemo, useState, useRef } from "react";
+import React, {
+  useEffect,
+  useMemo,
+  useState,
+  useRef,
+  useCallback,
+} from "react";
 import styles from "./Room.module.css";
 import { useParams, useLocation } from "react-router-dom";
 import { useVideoRoom } from "./ConferenceCallManagerHooks";
@@ -25,6 +31,7 @@ import {
   SettingsButton,
   MicButton,
   VideoButton,
+  LayoutToggleButton,
 } from "./RoomButton";
 import { Header, LeftNav, RightNav, CenterNav } from "./Header";
 import { Button } from "./Input";
@@ -71,6 +78,13 @@ export function Room({ manager }) {
     };
   }, []);
 
+  const [layout, setLayout] = useState("gallery");
+
+  const toggleLayout = useCallback(() => {
+    console.log(layout);
+    setLayout(layout === "spotlight" ? "gallery" : "spotlight");
+  }, [layout]);
+
   return (
     <div className={styles.room}>
       {!loading && room && (
@@ -80,6 +94,13 @@ export function Room({ manager }) {
             <h3>{room.name}</h3>
           </CenterNav>
           <RightNav>
+            {!loading && room && joined && (
+              <LayoutToggleButton
+                title={layout === "spotlight" ? "Spotlight" : "Gallery"}
+                layout={layout}
+                onClick={toggleLayout}
+              />
+            )}
             <SettingsButton
               title={debug ? "Disable DevTools" : "Enable DevTools"}
               on={debug}
@@ -111,7 +132,7 @@ export function Room({ manager }) {
         </div>
       )}
       {!loading && room && joined && participants.length > 0 && (
-        <VideoGrid participants={participants} />
+        <VideoGrid participants={participants} layout={layout} />
       )}
       {!loading && room && joined && (
         <div className={styles.footer}>

src/RoomButton.jsx

@@ -7,6 +7,8 @@ import { ReactComponent as VideoIcon } from "./icons/Video.svg";
 import { ReactComponent as DisableVideoIcon } from "./icons/DisableVideo.svg";
 import { ReactComponent as HangupIcon } from "./icons/Hangup.svg";
 import { ReactComponent as SettingsIcon } from "./icons/Settings.svg";
+import { ReactComponent as GridIcon } from "./icons/Grid.svg";
+import { ReactComponent as SpeakerIcon } from "./icons/Speaker.svg";
 
 export function RoomButton({ on, className, children, ...rest }) {
   return (
@@ -66,3 +68,15 @@ export function SettingsButton(props) {
     </HeaderButton>
   );
 }
+
+export function LayoutToggleButton({ layout, ...rest }) {
+  return (
+    <HeaderButton {...rest}>
+      {layout === "spotlight" ? (
+        <SpeakerIcon width={20} height={20} />
+      ) : (
+        <GridIcon width={20} height={20} />
+      )}
+    </HeaderButton>
+  );
+}

src/RoomButton.module.css

@@ -49,6 +49,10 @@ limitations under the License.
   border-radius: 32px;
 }
 
+.headerButton svg * {
+  fill: #8E99A4;
+}
+
 .headerButton:hover {
   background-color: #8D97A5;
 }

src/VideoGrid.jsx

@@ -403,13 +403,14 @@ function getSubGridPositions(tileCount, gridWidth, gridHeight, gap) {
   return newTilePositions;
 }
 
-export function VideoGrid({ participants }) {
+export function VideoGrid({ participants, layout }) {
   const [{ tiles, tilePositions }, setTileState] = useState({
     tiles: [],
     tilePositions: [],
   });
   const draggingTileRef = useRef(null);
   const lastTappedRef = useRef({});
+  const lastLayoutRef = useRef(layout);
   const isMounted = useIsMounted();
 
   const [gridRef, gridBounds] = useMeasure();
@@ -418,35 +419,34 @@ export function VideoGrid({ participants }) {
     setTileState(({ tiles }) => {
       const newTiles = [];
       const removedTileKeys = [];
-      let presenterTileCount = 0;
 
       for (const tile of tiles) {
-        const participant = participants.find(
+        let participant = participants.find(
           (participant) => participant.userId === tile.key
         );
 
-        if (tile.presenter) {
+        let remove = false;
-          presenterTileCount++;
+
+        if (!participant) {
+          remove = true;
+          participant = tile.participant;
+          removedTileKeys.push(tile.key);
         }
 
-        if (participant) {
+        let presenter;
-          // Existing tiles
+
-          newTiles.push({
+        if (layout === "spotlight") {
-            key: participant.userId,
+          presenter = participant.activeSpeaker;
-            participant: participant,
-            remove: false,
-            presenter: tile.presenter,
-          });
         } else {
-          // Removed tiles
+          presenter = layout === lastLayoutRef.current ? tile.presenter : false;
-          removedTileKeys.push(tile.key);
-          newTiles.push({
-            key: tile.key,
-            participant: tile.participant,
-            remove: true,
-            presenter: tile.presenter,
-          });
         }
+
+        newTiles.push({
+          key: participant.userId,
+          participant,
+          remove,
+          presenter,
+        });
       }
 
       for (const participant of participants) {
@@ -459,7 +459,7 @@ export function VideoGrid({ participants }) {
           key: participant.userId,
           participant,
           remove: false,
-          presenter: false,
+          presenter: layout === "spotlight" && participant.activeSpeaker,
         });
       }
 
@@ -476,6 +476,11 @@ export function VideoGrid({ participants }) {
               (tile) => !removedTileKeys.includes(tile.key)
             );
 
+            const presenterTileCount = newTiles.reduce(
+              (count, tile) => count + (tile.presenter ? 1 : 0),
+              0
+            );
+
             return {
               tiles: newTiles,
               tilePositions: getTilePositions(
@@ -489,6 +494,11 @@ export function VideoGrid({ participants }) {
         }, 250);
       }
 
+      const presenterTileCount = newTiles.reduce(
+        (count, tile) => count + (tile.presenter ? 1 : 0),
+        0
+      );
+
       return {
         tiles: newTiles,
         tilePositions: getTilePositions(
@@ -499,7 +509,7 @@ export function VideoGrid({ participants }) {
         ),
       };
     });
-  }, [participants, gridBounds]);
+  }, [participants, gridBounds, layout]);
 
   const animate = useCallback(
     (tiles) => (tileIndex) => {

src/icons/Grid.svg

@@ -1,11 +1,11 @@
 <svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
-<rect x="1" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="9" width="6" height="6" rx="1" fill="white"/>
-<rect x="1" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="17" width="6" height="6" rx="1" fill="white"/>
-<rect x="1" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="1" y="1" width="6" height="6" rx="1" fill="white"/>
-<rect x="9" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="9" y="1" width="6" height="6" rx="1" fill="white"/>
-<rect x="17" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
+<rect x="17" y="1" width="6" height="6" rx="1" fill="white"/>
 </svg>

src/icons/Speaker.svg

@@ -0,0 +1,6 @@
+<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
+<rect x="18" y="10" width="5" height="5" rx="1" fill="white"/>
+<rect x="18" y="16" width="5" height="5" rx="1" fill="white"/>
+<rect x="18" y="4" width="5" height="5" rx="1" fill="white"/>
+<rect x="1" y="4" width="16" height="17" rx="1" fill="white"/>
+</svg>