From f1ee3604de8be06e32bb16663aa0d5e1fcceab41 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 15 May 2023 22:03:26 -0400 Subject: [PATCH] Make Element Call work in Firefox's resist fingerprinting mode This one is gonna take some explaining: When in resist fingerprinting mode, Firefox exhibits some funny behavior: when we ask for the the list of media devices, it gives us fake device IDs. But when the js-sdk requests a stream for any of those devices, Firefox associates the stream with the real device ID. Now, in order to get the names of devices included in their metadata when you query the device list, you need to be holding a stream. For this reason, useMediaHandler was set up to reload the device list whenever matrix-js-sdk got a new local stream. But because of the inconsistency in device IDs, it would enter an infinite cycle telling matrix-js-sdk to request a stream for the fake device ID, but with matrix-js-sdk always responding with the real device ID. I already wasn't happy with useMediaHandler's use of @ts-ignore comments to inspect private js-sdk fields, and in the meantime we've come up with a simpler function for requesting device names, so I decided to refactor useMediaHandler to use it instead. Importantly, it doesn't break in resist fingerprinting mode. This created a new UX issue though: now, when on the lobby screen, useMediaHandler would request microphone access so it could get device names, followed immediately by a *second* pop-up for the lobby screen to request camera access. That's 1 pop-up too many, so I changed useMediaHandler to only request device names when a component is mounted that actually wants to show them. Currently, the settings modal is the only such component, and users normally only open it *after* granting full audio/video access, so this solution works out quite nicely. --- src/media-utils.ts | 2 +- src/room/GroupCallView.tsx | 4 +- src/settings/SettingsModal.tsx | 2 + src/settings/useMediaHandler.tsx | 203 +++++++++++++++---------------- 4 files changed, 104 insertions(+), 107 deletions(-) diff --git a/src/media-utils.ts b/src/media-utils.ts index 382360b..30bade9 100644 --- a/src/media-utils.ts +++ b/src/media-utils.ts @@ -47,7 +47,7 @@ export async function findDeviceByName( * * @return The available media devices */ -export async function getDevices(): Promise { +export async function getNamedDevices(): Promise { // First get the devices without their labels, to learn what kinds of streams // we can request let devices: MediaDeviceInfo[]; diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 1f20862..722686c 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -34,7 +34,7 @@ import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { useLocationNavigation } from "../useLocationNavigation"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useMediaHandler } from "../settings/useMediaHandler"; -import { findDeviceByName, getDevices } from "../media-utils"; +import { findDeviceByName, getNamedDevices } from "../media-utils"; declare global { interface Window { @@ -102,7 +102,7 @@ export function GroupCallView({ // Get the available devices so we can match the selected device // to its ID. This involves getting a media stream (see docs on // the function) so we only do it once and re-use the result. - const devices = await getDevices(); + const devices = await getNamedDevices(); const { audioInput, videoInput } = ev.detail .data as unknown as JoinCallData; diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index b126bb8..3b9db2e 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -57,7 +57,9 @@ export const SettingsModal = (props: Props) => { audioOutput, audioOutputs, setAudioOutput, + useDeviceNames, } = useMediaHandler(); + useDeviceNames(); const [spatialAudio, setSpatialAudio] = useSpatialAudio(); const [showInspector, setShowInspector] = useShowInspector(); diff --git a/src/settings/useMediaHandler.tsx b/src/settings/useMediaHandler.tsx index ce3953e..6bc3e56 100644 --- a/src/settings/useMediaHandler.tsx +++ b/src/settings/useMediaHandler.tsx @@ -32,7 +32,6 @@ limitations under the License. */ import { MatrixClient } from "matrix-js-sdk/src/client"; -import { MediaHandlerEvent } from "matrix-js-sdk/src/webrtc/mediaHandler"; import React, { useState, useEffect, @@ -41,18 +40,26 @@ import React, { useContext, createContext, ReactNode, + useRef, } from "react"; +import { getNamedDevices } from "../media-utils"; + export interface MediaHandlerContextInterface { - audioInput: string; + audioInput: string | undefined; audioInputs: MediaDeviceInfo[]; setAudioInput: (deviceId: string) => void; - videoInput: string; + videoInput: string | undefined; videoInputs: MediaDeviceInfo[]; setVideoInput: (deviceId: string) => void; - audioOutput: string; + audioOutput: string | undefined; audioOutputs: MediaDeviceInfo[]; setAudioOutput: (deviceId: string) => void; + /** + * A hook which requests for devices to be named. This requires media + * permissions. + */ + useDeviceNames: () => void; } const MediaHandlerContext = @@ -70,10 +77,10 @@ function getMediaPreferences(): MediaPreferences { try { return JSON.parse(mediaPreferences); } catch (e) { - return undefined; + return {}; } } else { - return undefined; + return {}; } } @@ -103,112 +110,98 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element { audioOutputs, }, setState, - ] = useState(() => { - const mediaPreferences = getMediaPreferences(); - const mediaHandler = client.getMediaHandler(); + ] = useState(() => ({ + audioInput: undefined as string | undefined, + videoInput: undefined as string | undefined, + audioOutput: undefined as string | undefined, + audioInputs: [] as MediaDeviceInfo[], + videoInputs: [] as MediaDeviceInfo[], + audioOutputs: [] as MediaDeviceInfo[], + })); - mediaHandler.restoreMediaSettings( - mediaPreferences?.audioInput, - mediaPreferences?.videoInput - ); + // A ref counting the number of components currently mounted that want + // to know device names + const numComponentsWantingNames = useRef(0); - return { - // @ts-ignore, ignore that audioInput is a private members of mediaHandler - audioInput: mediaHandler.audioInput, - // @ts-ignore, ignore that videoInput is a private members of mediaHandler - videoInput: mediaHandler.videoInput, - audioOutput: undefined, - audioInputs: [], - videoInputs: [], - audioOutputs: [], - }; - }); + const updateDevices = useCallback( + async (initial: boolean) => { + // Only request device names if components actually want them, because it + // could trigger an extra permission pop-up + const devices = await (numComponentsWantingNames.current > 0 + ? getNamedDevices() + : navigator.mediaDevices.enumerateDevices()); + const mediaPreferences = getMediaPreferences(); + + const audioInputs = devices.filter((d) => d.kind === "audioinput"); + const videoInputs = devices.filter((d) => d.kind === "videoinput"); + const audioOutputs = devices.filter((d) => d.kind === "audiooutput"); + + const audioInput = ( + mediaPreferences.audioInput === undefined + ? audioInputs.at(0) + : audioInputs.find( + (d) => d.deviceId === mediaPreferences.audioInput + ) ?? audioInputs.at(0) + )?.deviceId; + const videoInput = ( + mediaPreferences.videoInput === undefined + ? videoInputs.at(0) + : videoInputs.find( + (d) => d.deviceId === mediaPreferences.videoInput + ) ?? videoInputs.at(0) + )?.deviceId; + const audioOutput = + mediaPreferences.audioOutput === undefined + ? undefined + : audioOutputs.find( + (d) => d.deviceId === mediaPreferences.audioOutput + )?.deviceId; + + updateMediaPreferences({ audioInput, videoInput, audioOutput }); + setState({ + audioInput, + videoInput, + audioOutput, + audioInputs, + videoInputs, + audioOutputs, + }); + + if ( + initial || + audioInput !== mediaPreferences.audioInput || + videoInput !== mediaPreferences.videoInput + ) { + client.getMediaHandler().setMediaInputs(audioInput, videoInput); + } + }, + [client, setState] + ); + + const useDeviceNames = useCallback(() => { + // This is a little weird from React's perspective as it looks like a + // dynamic hook, but it works + // eslint-disable-next-line react-hooks/rules-of-hooks + useEffect(() => { + numComponentsWantingNames.current++; + if (numComponentsWantingNames.current === 1) updateDevices(false); + return () => void numComponentsWantingNames.current--; + }, []); + }, [updateDevices]); useEffect(() => { - const mediaHandler = client.getMediaHandler(); - - function updateDevices(): void { - navigator.mediaDevices.enumerateDevices().then((devices) => { - const mediaPreferences = getMediaPreferences(); - - const audioInputs = devices.filter( - (device) => device.kind === "audioinput" - ); - const audioConnected = audioInputs.some( - // @ts-ignore - (device) => device.deviceId === mediaHandler.audioInput - ); - // @ts-ignore - let audioInput = mediaHandler.audioInput; - - if (!audioConnected && audioInputs.length > 0) { - audioInput = audioInputs[0].deviceId; - } - - const videoInputs = devices.filter( - (device) => device.kind === "videoinput" - ); - const videoConnected = videoInputs.some( - // @ts-ignore - (device) => device.deviceId === mediaHandler.videoInput - ); - - // @ts-ignore - let videoInput = mediaHandler.videoInput; - - if (!videoConnected && videoInputs.length > 0) { - videoInput = videoInputs[0].deviceId; - } - - const audioOutputs = devices.filter( - (device) => device.kind === "audiooutput" - ); - let audioOutput = undefined; - - if ( - mediaPreferences && - audioOutputs.some( - (device) => device.deviceId === mediaPreferences.audioOutput - ) - ) { - audioOutput = mediaPreferences.audioOutput; - } - - if ( - // @ts-ignore - (mediaHandler.videoInput && mediaHandler.videoInput !== videoInput) || - // @ts-ignore - mediaHandler.audioInput !== audioInput - ) { - mediaHandler.setMediaInputs(audioInput, videoInput); - } - - updateMediaPreferences({ audioInput, videoInput, audioOutput }); - - setState({ - audioInput, - videoInput, - audioOutput, - audioInputs, - videoInputs, - audioOutputs, - }); - }); - } - updateDevices(); - - mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, updateDevices); - navigator.mediaDevices.addEventListener("devicechange", updateDevices); + updateDevices(true); + const onDeviceChange = () => updateDevices(false); + navigator.mediaDevices.addEventListener("devicechange", onDeviceChange); return () => { - mediaHandler.removeListener( - MediaHandlerEvent.LocalStreamsChanged, - updateDevices + navigator.mediaDevices.removeEventListener( + "devicechange", + onDeviceChange ); - navigator.mediaDevices.removeEventListener("devicechange", updateDevices); - mediaHandler.stopAllStreams(); + client.getMediaHandler().stopAllStreams(); }; - }, [client]); + }, [client, updateDevices]); const setAudioInput: (deviceId: string) => void = useCallback( (deviceId: string) => { @@ -245,6 +238,7 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element { audioOutput, audioOutputs, setAudioOutput, + useDeviceNames, }), [ audioInput, @@ -256,6 +250,7 @@ export function MediaHandlerProvider({ client, children }: Props): JSX.Element { audioOutput, audioOutputs, setAudioOutput, + useDeviceNames, ] );