diff --git a/package.json b/package.json index 3654051..83472e2 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ }, "dependencies": { "@juggle/resize-observer": "^3.3.1", + "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@react-aria/button": "^3.3.4", "@react-aria/dialog": "^3.1.4", "@react-aria/focus": "^3.5.0", diff --git a/src/IndexedDBWorker.js b/src/IndexedDBWorker.js new file mode 100644 index 0000000..0e373ca --- /dev/null +++ b/src/IndexedDBWorker.js @@ -0,0 +1,5 @@ +import { IndexedDBStoreWorker } from "matrix-js-sdk/src/indexeddb-worker"; + +const remoteWorker = new IndexedDBStoreWorker(self.postMessage); + +self.onmessage = remoteWorker.onMessage; diff --git a/src/matrix-utils.js b/src/matrix-utils.js index bdca07d..dba6f16 100644 --- a/src/matrix-utils.js +++ b/src/matrix-utils.js @@ -3,6 +3,9 @@ import { GroupCallIntent, GroupCallType, } from "matrix-js-sdk/src/browser-index"; +import IndexedDBWorker from "./IndexedDBWorker?worker"; +import Olm from "@matrix-org/olm"; +import olmWasmPath from "@matrix-org/olm/olm.wasm?url"; export const defaultHomeserver = import.meta.env.VITE_DEFAULT_HOMESERVER || @@ -26,11 +29,59 @@ function waitForSync(client) { } export async function initClient(clientOptions) { + // TODO: https://gitlab.matrix.org/matrix-org/olm/-/issues/10 + window.OLM_OPTIONS = {}; + await Olm.init({ locateFile: () => olmWasmPath }); + + let indexedDB; + + try { + indexedDB = window.indexedDB; + } catch (e) {} + + const storeOpts = {}; + + if (indexedDB && localStorage && !import.meta.env.DEV) { + storeOpts.store = new matrix.IndexedDBStore({ + indexedDB: window.indexedDB, + localStorage: window.localStorage, + dbName: "element-call-sync", + workerFactory: () => new IndexedDBWorker(), + }); + } + + if (localStorage) { + storeOpts.sessionStore = new matrix.WebStorageSessionStore(localStorage); + } + + if (indexedDB) { + storeOpts.cryptoStore = new matrix.IndexedDBCryptoStore( + indexedDB, + "matrix-js-sdk:crypto" + ); + } + const client = matrix.createClient({ + ...storeOpts, ...clientOptions, useAuthorizationHeader: true, }); + try { + await client.store.startup(); + } catch (error) { + console.error( + "Error starting matrix client store. Falling back to memory store.", + error + ); + client.store = new matrix.MemoryStore({ localStorage }); + await client.store.startup(); + } + + if (client.initCrypto) { + await client.initCrypto(); + } + await client.startClient({ // dirty hack to reduce chance of gappy syncs // should be fixed by spotting gaps and backpaginating diff --git a/src/room/PTTCallView.tsx b/src/room/PTTCallView.tsx index a06fb83..9087a26 100644 --- a/src/room/PTTCallView.tsx +++ b/src/room/PTTCallView.tsx @@ -110,8 +110,13 @@ export const PTTCallView: React.FC = ({ const { audioOutput } = useMediaHandler(); - const { startTalkingLocalRef, startTalkingRemoteRef, blockedRef, playClip } = - usePTTSounds(); + const { + startTalkingLocalRef, + startTalkingRemoteRef, + blockedRef, + endTalkingRef, + playClip, + } = usePTTSounds(); const { pttButtonHeld, @@ -153,6 +158,7 @@ export const PTTCallView: React.FC = ({ !f.isAudioMuted()); + + let activeSpeakerFeed = null; + let highestPowerLevel = null; + for (const feed of activeSpeakerFeeds) { + const member = groupCall.room.getMember(feed.userId); + if (highestPowerLevel === null || member.powerLevel > highestPowerLevel) { + highestPowerLevel = member.powerLevel; + activeSpeakerFeed = feed; + } + } + + return activeSpeakerFeed; +} + export interface PTTState { pttButtonHeld: boolean; isAdmin: boolean; @@ -40,6 +61,26 @@ export const usePTT = ( playClip: PlayClipFunction, enablePTTButton: boolean ): PTTState => { + // Used to serialise all the mute calls so they don't race. It has + // its own state as its always set separately from anything else. + const [mutePromise, setMutePromise] = useState( + Promise.resolve(false) + ); + + // Wrapper to serialise all the mute operations on the promise + const setMicMuteWrapper = useCallback( + (muted: boolean) => { + setMutePromise( + mutePromise.then(() => { + return groupCall.setMicrophoneMuted(muted).catch((e) => { + logger.error("Failed to unmute microphone", e); + }); + }) + ); + }, + [groupCall, mutePromise] + ); + const [ { pttButtonHeld, @@ -52,7 +93,7 @@ export const usePTT = ( ] = useState(() => { const roomMember = groupCall.room.getMember(client.getUserId()); - const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted()); + const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall); return { isAdmin: roomMember.powerLevel >= 100, @@ -63,38 +104,58 @@ export const usePTT = ( }; }); - useEffect(() => { - function onMuteStateChanged(...args): void { - const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted()); + const onMuteStateChanged = useCallback(() => { + const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall); - if (activeSpeakerUserId === null && activeSpeakerFeed.userId !== null) { - if (activeSpeakerFeed.userId === client.getUserId()) { - playClip(PTTClipID.START_TALKING_LOCAL); - } else { - playClip(PTTClipID.START_TALKING_REMOTE); - } - } else if ( - pttButtonHeld && - activeSpeakerUserId === client.getUserId() && - activeSpeakerFeed?.userId !== client.getUserId() - ) { - // We were talking but we've been cut off - playClip(PTTClipID.BLOCKED); + let blocked = false; + if (activeSpeakerUserId === null && activeSpeakerFeed !== null) { + if (activeSpeakerFeed.userId === client.getUserId()) { + playClip(PTTClipID.START_TALKING_LOCAL); + } else { + playClip(PTTClipID.START_TALKING_REMOTE); } + } else if (activeSpeakerUserId !== null && activeSpeakerFeed === null) { + playClip(PTTClipID.END_TALKING); + } else if ( + pttButtonHeld && + activeSpeakerUserId === client.getUserId() && + activeSpeakerFeed?.userId !== client.getUserId() + ) { + // We were talking but we've been cut off: mute our own mic + // (this is the easier way of cutting other speakers off if an + // admin barges in: we could also mute the non-admin speaker + // on all receivers, but we'd have to make sure we unmuted them + // correctly.) + setMicMuteWrapper(true); + blocked = true; + playClip(PTTClipID.BLOCKED); + } - setState((prevState) => ({ + setState((prevState) => { + return { ...prevState, activeSpeakerUserId: activeSpeakerFeed ? activeSpeakerFeed.userId : null, - })); - } + transmitBlocked: blocked, + }; + }); + }, [ + playClip, + groupCall, + pttButtonHeld, + activeSpeakerUserId, + client, + userMediaFeeds, + setMicMuteWrapper, + ]); + useEffect(() => { for (const callFeed of userMediaFeeds) { callFeed.addListener(CallFeedEvent.MuteStateChanged, onMuteStateChanged); } - const activeSpeakerFeed = userMediaFeeds.find((f) => !f.isAudioMuted()); + const activeSpeakerFeed = getActiveSpeakerFeed(userMediaFeeds, groupCall); setState((prevState) => ({ ...prevState, @@ -109,29 +170,26 @@ export const usePTT = ( ); } }; - }, [userMediaFeeds, activeSpeakerUserId, client, playClip, pttButtonHeld]); + }, [userMediaFeeds, onMuteStateChanged, groupCall]); const startTalking = useCallback(async () => { if (pttButtonHeld) return; let blocked = false; - if (!activeSpeakerUserId || (isAdmin && talkOverEnabled)) { - if (groupCall.isMicrophoneMuted()) { - try { - await groupCall.setMicrophoneMuted(false); - } catch (e) { - logger.error("Failed to unmute microphone", e); - } - } - } else { + if (activeSpeakerUserId && !(isAdmin && talkOverEnabled)) { playClip(PTTClipID.BLOCKED); blocked = true; } + // setstate before doing the async call to mute / unmute the mic setState((prevState) => ({ ...prevState, pttButtonHeld: true, transmitBlocked: blocked, })); + + if (!blocked && groupCall.isMicrophoneMuted()) { + setMicMuteWrapper(false); + } }, [ pttButtonHeld, groupCall, @@ -140,25 +198,18 @@ export const usePTT = ( talkOverEnabled, setState, playClip, + setMicMuteWrapper, ]); - const stopTalking = useCallback(() => { - setState((prevState) => ({ - ...prevState, - pttButtonHeld: false, - unmuteError: null, - })); - - if (!groupCall.isMicrophoneMuted()) { - groupCall.setMicrophoneMuted(true); - } - + const stopTalking = useCallback(async () => { setState((prevState) => ({ ...prevState, pttButtonHeld: false, transmitBlocked: false, })); - }, [groupCall]); + + setMicMuteWrapper(true); + }, [setMicMuteWrapper]); useEffect(() => { function onKeyDown(event: KeyboardEvent): void { @@ -184,7 +235,7 @@ export const usePTT = ( function onBlur(): void { // TODO: We will need to disable this for a global PTT hotkey to work if (!groupCall.isMicrophoneMuted()) { - groupCall.setMicrophoneMuted(true); + setMicMuteWrapper(true); } setState((prevState) => ({ ...prevState, pttButtonHeld: false })); @@ -208,6 +259,7 @@ export const usePTT = ( talkOverEnabled, pttButtonHeld, enablePTTButton, + setMicMuteWrapper, ]); const setTalkOverEnabled = useCallback((talkOverEnabled) => { diff --git a/src/sound/PTTClips.tsx b/src/sound/PTTClips.tsx index e13acb5..90958af 100644 --- a/src/sound/PTTClips.tsx +++ b/src/sound/PTTClips.tsx @@ -20,6 +20,8 @@ import startTalkLocalOggUrl from "./start_talk_local.ogg"; import startTalkLocalMp3Url from "./start_talk_local.mp3"; import startTalkRemoteOggUrl from "./start_talk_remote.ogg"; import startTalkRemoteMp3Url from "./start_talk_remote.mp3"; +import endTalkOggUrl from "./end_talk.ogg"; +import endTalkMp3Url from "./end_talk.mp3"; import blockedOggUrl from "./blocked.ogg"; import blockedMp3Url from "./blocked.mp3"; import styles from "./PTTClips.module.css"; @@ -27,12 +29,14 @@ import styles from "./PTTClips.module.css"; interface Props { startTalkingLocalRef: React.RefObject; startTalkingRemoteRef: React.RefObject; + endTalkingRef: React.RefObject; blockedRef: React.RefObject; } export const PTTClips: React.FC = ({ startTalkingLocalRef, startTalkingRemoteRef, + endTalkingRef, blockedRef, }) => { return ( @@ -53,6 +57,10 @@ export const PTTClips: React.FC = ({ +