Finish active speaker selection

public/browser-matrix.js

@@ -44766,7 +44766,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
+exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.CallFeed = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
 const memory_crypto_store_1 = require("./crypto/store/memory-crypto-store");
 const memory_1 = require("./store/memory");
 const scheduler_1 = require("./scheduler");
@@ -44800,6 +44800,11 @@ var call_1 = require("./webrtc/call");
 Object.defineProperty(exports, "createNewMatrixCall", { enumerable: true, get: function () { return call_1.createNewMatrixCall; } });
 Object.defineProperty(exports, "setMatrixCallAudioInput", { enumerable: true, get: function () { return call_1.setAudioInput; } });
 Object.defineProperty(exports, "setMatrixCallVideoInput", { enumerable: true, get: function () { return call_1.setVideoInput; } });
+// TODO: This export is temporary and is only used for the local call feed for conference calls
+// Ideally conference calls will become a first-class concept and we will have a local call feed with
+// a lifecycle that matches the conference call, not individual calls to members.
+var callFeed_1 = require("./webrtc/callFeed");
+Object.defineProperty(exports, "CallFeed", { enumerable: true, get: function () { return callFeed_1.CallFeed; } });
 // expose the underlying request object so different environments can use
 // different request libs (e.g. request or browser-request)
 let requestInstance;
@@ -44922,7 +44927,7 @@ exports.createClient = createClient;
 
 }).call(this)}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {})
 
-},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151}],120:[function(require,module,exports){
+},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151,"./webrtc/callFeed":154}],120:[function(require,module,exports){
 "use strict";
 /*
 Copyright 2021 The Matrix.org Foundation C.I.C.
@@ -59157,6 +59162,9 @@ class MatrixCall extends events_1.EventEmitter {
         logger_1.logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`);
     }
     deleteAllFeeds() {
+        for (const feed of this.feeds) {
+            feed.dispose();
+        }
         this.feeds = [];
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -59167,6 +59175,7 @@ class MatrixCall extends events_1.EventEmitter {
             logger_1.logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`);
             return;
         }
+        feed.dispose();
         this.feeds.splice(this.feeds.indexOf(feed), 1);
         this.emit(CallEvent.FeedsChanged, this.feeds);
     }
@@ -60737,7 +60746,7 @@ class CallFeed extends events_1.default {
     volumeLooper() {
         if (!this.analyser)
             return;
-        setTimeout(() => {
+        this.volumeLooperTimeout = setTimeout(() => {
             if (!this.measuringVolumeActivity)
                 return;
             this.analyser.getFloatFrequencyData(this.frequencyBinCount);
@@ -60756,6 +60765,9 @@ class CallFeed extends events_1.default {
             this.volumeLooper();
         }, POLLING_INTERVAL);
     }
+    dispose() {
+        clearTimeout(this.volumeLooperTimeout);
+    }
 }
 exports.CallFeed = CallFeed;
 

src/ConferenceCallManager.js

@@ -22,6 +22,8 @@ const CONF_ROOM = "me.robertlong.conf";
 const CONF_PARTICIPANT = "me.robertlong.conf.participant";
 const PARTICIPANT_TIMEOUT = 1000 * 15;
 const SPEAKING_THRESHOLD = -80;
+const ACTIVE_SPEAKER_INTERVAL = 1000;
+const ACTIVE_SPEAKER_SAMPLES = 8;
 
 function waitForSync(client) {
   return new Promise((resolve, reject) => {
@@ -172,6 +174,7 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.localVideoStream = null;
     this.localParticipant = null;
+    this.localCallFeed = null;
 
     this.audioMuted = false;
     this.videoMuted = false;
@@ -261,6 +264,30 @@ export class ConferenceCallManager extends EventEmitter {
     this.participants.push(this.localParticipant);
     this.emit("debugstate", userId, null, "you");
 
+    this.localCallFeed = new matrixcs.CallFeed(
+      stream,
+      this.localParticipant.userId,
+      "m.usermedia",
+      this.client,
+      this.room.roomId,
+      this.audioMuted,
+      this.videoMuted
+    );
+    this.localCallFeed.on("mute_state_changed", () =>
+      this._onCallFeedMuteStateChanged(
+        this.localParticipant,
+        this.localCallFeed
+      )
+    );
+    this.localCallFeed.setSpeakingThreshold(SPEAKING_THRESHOLD);
+    this.localCallFeed.measureVolumeActivity(true);
+    this.localCallFeed.on("speaking", (speaking) => {
+      this._onCallFeedSpeaking(this.localParticipant, speaking);
+    });
+    this.localCallFeed.on("volume_changed", (maxVolume) =>
+      this._onCallFeedVolumeChange(this.localParticipant, maxVolume)
+    );
+
     // Announce to the other room members that we have entered the room.
     // Continue doing so every PARTICIPANT_TIMEOUT ms
     this._updateMemberParticipantState();
@@ -285,7 +312,6 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.emit("entered");
     this.emit("participants_changed");
-    console.log("enter");
     this._onActiveSpeakerLoop();
   }
 
@@ -318,6 +344,8 @@ export class ConferenceCallManager extends EventEmitter {
 
     this.client.stopLocalMediaStream();
     this.localVideoStream = null;
+    this.localCallFeed.dispose();
+    this.localCallFeed = null;
 
     this.room = null;
     this.entered = false;
@@ -329,6 +357,7 @@ export class ConferenceCallManager extends EventEmitter {
     this.videoMuted = false;
     clearTimeout(this._memberParticipantStateTimeout);
     clearTimeout(this._activeSpeakerLoopTimeout);
+    this._speakerMap.clear();
 
     this.emit("participants_changed");
     this.emit("left");
@@ -349,8 +378,8 @@ export class ConferenceCallManager extends EventEmitter {
   setAudioMuted(muted) {
     this.audioMuted = muted;
 
-    if (this.localParticipant) {
+    if (this.localCallFeed) {
-      this.localParticipant.audioMuted = muted;
+      this.localCallFeed.setAudioMuted(muted);
     }
 
     const localStream = this.localVideoStream;
@@ -381,8 +410,8 @@ export class ConferenceCallManager extends EventEmitter {
   setVideoMuted(muted) {
     this.videoMuted = muted;
 
-    if (this.localParticipant) {
+    if (this.localCallFeed) {
-      this.localParticipant.videoMuted = muted;
+      this.localCallFeed.setVideoMuted(muted);
     }
 
     const localStream = this.localVideoStream;
@@ -734,12 +763,10 @@ export class ConferenceCallManager extends EventEmitter {
     participant.videoMuted = feed.isVideoMuted();
 
     if (participant.audioMuted) {
-      this._speakerMap.set(participant.userId, [
+      this._speakerMap.set(
-        -Infinity,
+        participant.userId,
-        -Infinity,
+        Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
-        -Infinity,
+      );
-        -Infinity,
-      ]);
     }
 
     this.emit("participants_changed");
@@ -752,12 +779,10 @@ export class ConferenceCallManager extends EventEmitter {
 
   _onCallFeedVolumeChange = (participant, maxVolume) => {
     if (!this._speakerMap.has(participant.userId)) {
-      this._speakerMap.set(participant.userId, [
+      this._speakerMap.set(
-        -Infinity,
+        participant.userId,
-        -Infinity,
+        Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
-        -Infinity,
+      );
-        -Infinity,
-      ]);
     }
 
     const volumeArr = this._speakerMap.get(participant.userId);
@@ -777,7 +802,7 @@ export class ConferenceCallManager extends EventEmitter {
         total += Math.max(volume, SPEAKING_THRESHOLD);
       }
 
-      const avg = total / 4;
+      const avg = total / ACTIVE_SPEAKER_SAMPLES;
 
       if (!topAvg) {
         topAvg = avg;
@@ -788,7 +813,7 @@ export class ConferenceCallManager extends EventEmitter {
       }
     }
 
-    if (activeSpeakerId) {
+    if (activeSpeakerId && topAvg > SPEAKING_THRESHOLD) {
       const nextActiveSpeaker = this.participants.find(
         (p) => p.userId === activeSpeakerId
       );
@@ -797,12 +822,14 @@ export class ConferenceCallManager extends EventEmitter {
         this.activeSpeaker.activeSpeaker = false;
         nextActiveSpeaker.activeSpeaker = true;
         this.activeSpeaker = nextActiveSpeaker;
-        console.log("activeSpeakerChanged", this.activeSpeaker);
         this.emit("participants_changed");
       }
     }
 
-    this._activeSpeakerLoopTimeout = setTimeout(this._onActiveSpeakerLoop, 250);
+    this._activeSpeakerLoopTimeout = setTimeout(
+      this._onActiveSpeakerLoop,
+      ACTIVE_SPEAKER_INTERVAL
+    );
   };
 
   _onCallReplaced = (participant, call, newCall) => {

src/Room.jsx

@@ -81,7 +81,6 @@ export function Room({ manager }) {
   const [layout, setLayout] = useState("gallery");
 
   const toggleLayout = useCallback(() => {
-    console.log(layout);
     setLayout(layout === "spotlight" ? "gallery" : "spotlight");
   }, [layout]);