Merge pull request #24 from vector-im/feature/active-speaker-layout

Active Speaker Layout
This commit is contained in:
Robert Long 2021-09-02 13:44:05 -07:00 committed by GitHub
commit 196c8eeeeb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 242 additions and 51 deletions

View file

@ -44766,7 +44766,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
return result;
};
Object.defineProperty(exports, "__esModule", { value: true });
exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
exports.createClient = exports.setCryptoStoreFactory = exports.wrapRequest = exports.getRequest = exports.request = exports.CallFeed = exports.setMatrixCallVideoInput = exports.setMatrixCallAudioInput = exports.createNewMatrixCall = exports.ContentHelpers = void 0;
const memory_crypto_store_1 = require("./crypto/store/memory-crypto-store");
const memory_1 = require("./store/memory");
const scheduler_1 = require("./scheduler");
@ -44800,6 +44800,11 @@ var call_1 = require("./webrtc/call");
Object.defineProperty(exports, "createNewMatrixCall", { enumerable: true, get: function () { return call_1.createNewMatrixCall; } });
Object.defineProperty(exports, "setMatrixCallAudioInput", { enumerable: true, get: function () { return call_1.setAudioInput; } });
Object.defineProperty(exports, "setMatrixCallVideoInput", { enumerable: true, get: function () { return call_1.setVideoInput; } });
// TODO: This export is temporary and is only used for the local call feed for conference calls
// Ideally conference calls will become a first-class concept and we will have a local call feed with
// a lifecycle that matches the conference call, not individual calls to members.
var callFeed_1 = require("./webrtc/callFeed");
Object.defineProperty(exports, "CallFeed", { enumerable: true, get: function () { return callFeed_1.CallFeed; } });
// expose the underlying request object so different environments can use
// different request libs (e.g. request or browser-request)
let requestInstance;
@ -44922,7 +44927,7 @@ exports.createClient = createClient;
}).call(this)}).call(this,typeof global !== "undefined" ? global : typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {})
},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151}],120:[function(require,module,exports){
},{"./autodiscovery":74,"./client":76,"./content-helpers":77,"./content-repo":78,"./crypto/store/indexeddb-crypto-store":100,"./crypto/store/memory-crypto-store":102,"./errors":111,"./filter":114,"./http-api":115,"./interactive-auth":117,"./models/event":125,"./models/event-timeline":124,"./models/event-timeline-set":123,"./models/group":126,"./models/room":131,"./models/room-member":128,"./models/room-state":129,"./models/user":134,"./scheduler":138,"./service-types":139,"./store/indexeddb":142,"./store/memory":143,"./store/session/webstorage":144,"./sync-accumulator":146,"./timeline-window":149,"./webrtc/call":151,"./webrtc/callFeed":154}],120:[function(require,module,exports){
"use strict";
/*
Copyright 2021 The Matrix.org Foundation C.I.C.
@ -59157,6 +59162,9 @@ class MatrixCall extends events_1.EventEmitter {
logger_1.logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`);
}
deleteAllFeeds() {
for (const feed of this.feeds) {
feed.dispose();
}
this.feeds = [];
this.emit(CallEvent.FeedsChanged, this.feeds);
}
@ -59167,6 +59175,7 @@ class MatrixCall extends events_1.EventEmitter {
logger_1.logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`);
return;
}
feed.dispose();
this.feeds.splice(this.feeds.indexOf(feed), 1);
this.emit(CallEvent.FeedsChanged, this.feeds);
}
@ -60737,7 +60746,7 @@ class CallFeed extends events_1.default {
volumeLooper() {
if (!this.analyser)
return;
setTimeout(() => {
this.volumeLooperTimeout = setTimeout(() => {
if (!this.measuringVolumeActivity)
return;
this.analyser.getFloatFrequencyData(this.frequencyBinCount);
@ -60756,6 +60765,9 @@ class CallFeed extends events_1.default {
this.volumeLooper();
}, POLLING_INTERVAL);
}
dispose() {
clearTimeout(this.volumeLooperTimeout);
}
}
exports.CallFeed = CallFeed;

File diff suppressed because one or more lines are too long

View file

@ -22,6 +22,8 @@ const CONF_ROOM = "me.robertlong.conf";
const CONF_PARTICIPANT = "me.robertlong.conf.participant";
const PARTICIPANT_TIMEOUT = 1000 * 15;
const SPEAKING_THRESHOLD = -80;
const ACTIVE_SPEAKER_INTERVAL = 1000;
const ACTIVE_SPEAKER_SAMPLES = 8;
function waitForSync(client) {
return new Promise((resolve, reject) => {
@ -172,10 +174,15 @@ export class ConferenceCallManager extends EventEmitter {
this.localVideoStream = null;
this.localParticipant = null;
this.localCallFeed = null;
this.audioMuted = false;
this.videoMuted = false;
this.activeSpeaker = null;
this._speakerMap = new Map();
this._activeSpeakerLoopTimeout = null;
this.client.on("RoomState.members", this._onRoomStateMembers);
this.client.on("Call.incoming", this._onIncomingCall);
this.callDebugger = new ConferenceCallDebugger(this);
@ -249,11 +256,38 @@ export class ConferenceCallManager extends EventEmitter {
audioMuted: this.audioMuted,
videoMuted: this.videoMuted,
speaking: false,
activeSpeaker: true,
};
this.activeSpeaker = this.localParticipant;
this.participants.push(this.localParticipant);
this.emit("debugstate", userId, null, "you");
this.localCallFeed = new matrixcs.CallFeed(
stream,
this.localParticipant.userId,
"m.usermedia",
this.client,
this.room.roomId,
this.audioMuted,
this.videoMuted
);
this.localCallFeed.on("mute_state_changed", () =>
this._onCallFeedMuteStateChanged(
this.localParticipant,
this.localCallFeed
)
);
this.localCallFeed.setSpeakingThreshold(SPEAKING_THRESHOLD);
this.localCallFeed.measureVolumeActivity(true);
this.localCallFeed.on("speaking", (speaking) => {
this._onCallFeedSpeaking(this.localParticipant, speaking);
});
this.localCallFeed.on("volume_changed", (maxVolume) =>
this._onCallFeedVolumeChange(this.localParticipant, maxVolume)
);
// Announce to the other room members that we have entered the room.
// Continue doing so every PARTICIPANT_TIMEOUT ms
this._updateMemberParticipantState();
@ -278,6 +312,7 @@ export class ConferenceCallManager extends EventEmitter {
this.emit("entered");
this.emit("participants_changed");
this._onActiveSpeakerLoop();
}
leaveCall() {
@ -309,19 +344,20 @@ export class ConferenceCallManager extends EventEmitter {
this.client.stopLocalMediaStream();
this.localVideoStream = null;
this.localCallFeed.dispose();
this.localCallFeed = null;
this.room = null;
this.entered = false;
this._left = true;
this.participants = [];
this.localParticipant.stream = null;
this.localParticipant.call = null;
this.localParticipant.audioMuted = false;
this.localParticipant.videoMuted = false;
this.localParticipant.speaking = false;
this.localParticipant = null;
this.activeSpeaker = null;
this.audioMuted = false;
this.videoMuted = false;
clearTimeout(this._memberParticipantStateTimeout);
clearTimeout(this._activeSpeakerLoopTimeout);
this._speakerMap.clear();
this.emit("participants_changed");
this.emit("left");
@ -342,8 +378,8 @@ export class ConferenceCallManager extends EventEmitter {
setAudioMuted(muted) {
this.audioMuted = muted;
if (this.localParticipant) {
this.localParticipant.audioMuted = muted;
if (this.localCallFeed) {
this.localCallFeed.setAudioMuted(muted);
}
const localStream = this.localVideoStream;
@ -374,8 +410,8 @@ export class ConferenceCallManager extends EventEmitter {
setVideoMuted(muted) {
this.videoMuted = muted;
if (this.localParticipant) {
this.localParticipant.videoMuted = muted;
if (this.localCallFeed) {
this.localCallFeed.setVideoMuted(muted);
}
const localStream = this.localVideoStream;
@ -506,6 +542,9 @@ export class ConferenceCallManager extends EventEmitter {
remoteFeed.on("speaking", (speaking) => {
this._onCallFeedSpeaking(participant, speaking);
});
remoteFeed.on("volume_changed", (maxVolume) =>
this._onCallFeedVolumeChange(participant, maxVolume)
);
}
const userId = call.opponentMember.userId;
@ -532,6 +571,7 @@ export class ConferenceCallManager extends EventEmitter {
existingParticipant.audioMuted = audioMuted;
existingParticipant.videoMuted = videoMuted;
existingParticipant.speaking = false;
existingParticipant.activeSpeaker = false;
existingParticipant.sessionId = sessionId;
} else {
participant = {
@ -543,6 +583,7 @@ export class ConferenceCallManager extends EventEmitter {
audioMuted,
videoMuted,
speaking: false,
activeSpeaker: false,
};
this.participants.push(participant);
}
@ -634,6 +675,7 @@ export class ConferenceCallManager extends EventEmitter {
participant.audioMuted = false;
participant.videoMuted = false;
participant.speaking = false;
participant.activeSpeaker = false;
} else {
participant = {
local: false,
@ -644,6 +686,7 @@ export class ConferenceCallManager extends EventEmitter {
audioMuted: false,
videoMuted: false,
speaking: false,
activeSpeaker: false,
};
// TODO: Should we wait until the call has been answered to push the participant?
// Or do we hide the participant until their stream is live?
@ -708,6 +751,9 @@ export class ConferenceCallManager extends EventEmitter {
remoteFeed.on("speaking", (speaking) => {
this._onCallFeedSpeaking(participant, speaking);
});
remoteFeed.on("volume_changed", (maxVolume) =>
this._onCallFeedVolumeChange(participant, maxVolume)
);
this._onCallFeedMuteStateChanged(participant, remoteFeed);
}
};
@ -715,6 +761,14 @@ export class ConferenceCallManager extends EventEmitter {
_onCallFeedMuteStateChanged = (participant, feed) => {
participant.audioMuted = feed.isAudioMuted();
participant.videoMuted = feed.isVideoMuted();
if (participant.audioMuted) {
this._speakerMap.set(
participant.userId,
Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
);
}
this.emit("participants_changed");
};
@ -723,6 +777,61 @@ export class ConferenceCallManager extends EventEmitter {
this.emit("participants_changed");
};
_onCallFeedVolumeChange = (participant, maxVolume) => {
if (!this._speakerMap.has(participant.userId)) {
this._speakerMap.set(
participant.userId,
Array(ACTIVE_SPEAKER_SAMPLES).fill(-Infinity)
);
}
const volumeArr = this._speakerMap.get(participant.userId);
volumeArr.shift();
volumeArr.push(maxVolume);
};
_onActiveSpeakerLoop = () => {
let topAvg;
let activeSpeakerId;
for (const [userId, volumeArr] of this._speakerMap) {
let total = 0;
for (let i = 0; i < volumeArr.length; i++) {
const volume = volumeArr[i];
total += Math.max(volume, SPEAKING_THRESHOLD);
}
const avg = total / ACTIVE_SPEAKER_SAMPLES;
if (!topAvg) {
topAvg = avg;
activeSpeakerId = userId;
} else if (avg > topAvg) {
topAvg = avg;
activeSpeakerId = userId;
}
}
if (activeSpeakerId && topAvg > SPEAKING_THRESHOLD) {
const nextActiveSpeaker = this.participants.find(
(p) => p.userId === activeSpeakerId
);
if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) {
this.activeSpeaker.activeSpeaker = false;
nextActiveSpeaker.activeSpeaker = true;
this.activeSpeaker = nextActiveSpeaker;
this.emit("participants_changed");
}
}
this._activeSpeakerLoopTimeout = setTimeout(
this._onActiveSpeakerLoop,
ACTIVE_SPEAKER_INTERVAL
);
};
_onCallReplaced = (participant, call, newCall) => {
participant.call = newCall;
@ -751,6 +860,9 @@ export class ConferenceCallManager extends EventEmitter {
remoteFeed.on("speaking", (speaking) => {
this._onCallFeedSpeaking(participant, speaking);
});
remoteFeed.on("volume_changed", (maxVolume) =>
this._onCallFeedVolumeChange(participant, maxVolume)
);
}
this.emit("call", newCall);
@ -770,6 +882,13 @@ export class ConferenceCallManager extends EventEmitter {
this.participants.splice(participantIndex, 1);
if (this.activeSpeaker === participant && this.participants.length > 0) {
this.activeSpeaker = this.participants[0];
this.activeSpeaker.activeSpeaker = true;
}
this._speakerMap.delete(participant.userId);
this.emit("participants_changed");
};

View file

@ -14,7 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useEffect, useMemo, useState, useRef } from "react";
import React, {
useEffect,
useMemo,
useState,
useRef,
useCallback,
} from "react";
import styles from "./Room.module.css";
import { useParams, useLocation } from "react-router-dom";
import { useVideoRoom } from "./ConferenceCallManagerHooks";
@ -25,6 +31,7 @@ import {
SettingsButton,
MicButton,
VideoButton,
LayoutToggleButton,
} from "./RoomButton";
import { Header, LeftNav, RightNav, CenterNav } from "./Header";
import { Button } from "./Input";
@ -71,6 +78,12 @@ export function Room({ manager }) {
};
}, []);
const [layout, setLayout] = useState("gallery");
const toggleLayout = useCallback(() => {
setLayout(layout === "spotlight" ? "gallery" : "spotlight");
}, [layout]);
return (
<div className={styles.room}>
{!loading && room && (
@ -80,6 +93,13 @@ export function Room({ manager }) {
<h3>{room.name}</h3>
</CenterNav>
<RightNav>
{!loading && room && joined && (
<LayoutToggleButton
title={layout === "spotlight" ? "Spotlight" : "Gallery"}
layout={layout}
onClick={toggleLayout}
/>
)}
<SettingsButton
title={debug ? "Disable DevTools" : "Enable DevTools"}
on={debug}
@ -111,7 +131,7 @@ export function Room({ manager }) {
</div>
)}
{!loading && room && joined && participants.length > 0 && (
<VideoGrid participants={participants} />
<VideoGrid participants={participants} layout={layout} />
)}
{!loading && room && joined && (
<div className={styles.footer}>

View file

@ -7,6 +7,8 @@ import { ReactComponent as VideoIcon } from "./icons/Video.svg";
import { ReactComponent as DisableVideoIcon } from "./icons/DisableVideo.svg";
import { ReactComponent as HangupIcon } from "./icons/Hangup.svg";
import { ReactComponent as SettingsIcon } from "./icons/Settings.svg";
import { ReactComponent as GridIcon } from "./icons/Grid.svg";
import { ReactComponent as SpeakerIcon } from "./icons/Speaker.svg";
export function RoomButton({ on, className, children, ...rest }) {
return (
@ -66,3 +68,15 @@ export function SettingsButton(props) {
</HeaderButton>
);
}
export function LayoutToggleButton({ layout, ...rest }) {
return (
<HeaderButton {...rest}>
{layout === "spotlight" ? (
<SpeakerIcon width={20} height={20} />
) : (
<GridIcon width={20} height={20} />
)}
</HeaderButton>
);
}

View file

@ -49,6 +49,10 @@ limitations under the License.
border-radius: 32px;
}
.headerButton svg * {
fill: #8E99A4;
}
.headerButton:hover {
background-color: #8D97A5;
}

View file

@ -403,13 +403,14 @@ function getSubGridPositions(tileCount, gridWidth, gridHeight, gap) {
return newTilePositions;
}
export function VideoGrid({ participants }) {
export function VideoGrid({ participants, layout }) {
const [{ tiles, tilePositions }, setTileState] = useState({
tiles: [],
tilePositions: [],
});
const draggingTileRef = useRef(null);
const lastTappedRef = useRef({});
const lastLayoutRef = useRef(layout);
const isMounted = useIsMounted();
const [gridRef, gridBounds] = useMeasure();
@ -418,35 +419,34 @@ export function VideoGrid({ participants }) {
setTileState(({ tiles }) => {
const newTiles = [];
const removedTileKeys = [];
let presenterTileCount = 0;
for (const tile of tiles) {
const participant = participants.find(
let participant = participants.find(
(participant) => participant.userId === tile.key
);
if (tile.presenter) {
presenterTileCount++;
let remove = false;
if (!participant) {
remove = true;
participant = tile.participant;
removedTileKeys.push(tile.key);
}
if (participant) {
// Existing tiles
newTiles.push({
key: participant.userId,
participant: participant,
remove: false,
presenter: tile.presenter,
});
let presenter;
if (layout === "spotlight") {
presenter = participant.activeSpeaker;
} else {
// Removed tiles
removedTileKeys.push(tile.key);
newTiles.push({
key: tile.key,
participant: tile.participant,
remove: true,
presenter: tile.presenter,
});
presenter = layout === lastLayoutRef.current ? tile.presenter : false;
}
newTiles.push({
key: participant.userId,
participant,
remove,
presenter,
});
}
for (const participant of participants) {
@ -459,7 +459,7 @@ export function VideoGrid({ participants }) {
key: participant.userId,
participant,
remove: false,
presenter: false,
presenter: layout === "spotlight" && participant.activeSpeaker,
});
}
@ -476,6 +476,11 @@ export function VideoGrid({ participants }) {
(tile) => !removedTileKeys.includes(tile.key)
);
const presenterTileCount = newTiles.reduce(
(count, tile) => count + (tile.presenter ? 1 : 0),
0
);
return {
tiles: newTiles,
tilePositions: getTilePositions(
@ -489,6 +494,13 @@ export function VideoGrid({ participants }) {
}, 250);
}
const presenterTileCount = newTiles.reduce(
(count, tile) => count + (tile.presenter ? 1 : 0),
0
);
lastLayoutRef.current = layout;
return {
tiles: newTiles,
tilePositions: getTilePositions(
@ -499,7 +511,7 @@ export function VideoGrid({ participants }) {
),
};
});
}, [participants, gridBounds]);
}, [participants, gridBounds, layout]);
const animate = useCallback(
(tiles) => (tileIndex) => {
@ -673,7 +685,7 @@ export function VideoGrid({ participants }) {
api.start(animate(newTiles));
},
{ filterTaps: true }
{ filterTaps: true, enabled: layout === "gallery" }
);
return (
@ -699,6 +711,10 @@ export function VideoGrid({ participants }) {
);
}
VideoGrid.defaultProps = {
layout: "gallery",
};
function ParticipantTile({ style, participant, remove, presenter, ...rest }) {
const videoRef = useRef();

View file

@ -1,11 +1,11 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect x="1" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="9" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="17" y="9" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="1" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="9" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="17" y="17" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="1" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="9" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="17" y="1" width="6" height="6" rx="1" fill="#8E99A4"/>
<rect x="1" y="9" width="6" height="6" rx="1" fill="white"/>
<rect x="9" y="9" width="6" height="6" rx="1" fill="white"/>
<rect x="17" y="9" width="6" height="6" rx="1" fill="white"/>
<rect x="1" y="17" width="6" height="6" rx="1" fill="white"/>
<rect x="9" y="17" width="6" height="6" rx="1" fill="white"/>
<rect x="17" y="17" width="6" height="6" rx="1" fill="white"/>
<rect x="1" y="1" width="6" height="6" rx="1" fill="white"/>
<rect x="9" y="1" width="6" height="6" rx="1" fill="white"/>
<rect x="17" y="1" width="6" height="6" rx="1" fill="white"/>
</svg>

Before

Width:  |  Height:  |  Size: 676 B

After

Width:  |  Height:  |  Size: 658 B

6
src/icons/Speaker.svg Normal file
View file

@ -0,0 +1,6 @@
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
<rect x="18" y="10" width="5" height="5" rx="1" fill="white"/>
<rect x="18" y="16" width="5" height="5" rx="1" fill="white"/>
<rect x="18" y="4" width="5" height="5" rx="1" fill="white"/>
<rect x="1" y="4" width="16" height="17" rx="1" fill="white"/>
</svg>

After

Width:  |  Height:  |  Size: 354 B