From 4a5b69800ccb6de3ac0a5b4a6244ab781274445a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Sep 2022 16:19:48 +0100 Subject: [PATCH 1/4] Use device labels rather than IDs in widget API device IDs are different for each origin, so won't match up when passed in & out of widgets. Use the label instead. For https://github.com/vector-im/element-web/issues/23331 --- src/media-utils.ts | 23 +++++++++++++++++++++++ src/room/GroupCallView.tsx | 28 ++++++++++++++++++++++++++-- 2 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 src/media-utils.ts diff --git a/src/media-utils.ts b/src/media-utils.ts new file mode 100644 index 0000000..0c01b5d --- /dev/null +++ b/src/media-utils.ts @@ -0,0 +1,23 @@ +/* +Copyright 2022 Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export async function findDeviceByName( + deviceName: string +): Promise { + const devices = await navigator.mediaDevices.enumerateDevices(); + const deviceInfo = devices.find((d) => d.label === deviceName); + return deviceInfo?.deviceId; +} diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 0ee6cfb..0598e4d 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -18,6 +18,7 @@ import React, { useCallback, useEffect, useState } from "react"; import { useHistory } from "react-router-dom"; import { GroupCall, GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall"; import { MatrixClient } from "matrix-js-sdk/src/client"; +import { logger } from "matrix-js-sdk/src/logger"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { widget, ElementWidgetActions, JoinCallData } from "../widget"; @@ -31,6 +32,7 @@ import { useRoomAvatar } from "./useRoomAvatar"; import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { useLocationNavigation } from "../useLocationNavigation"; import { useMediaHandler } from "../settings/useMediaHandler"; +import { findDeviceByName } from "../media-utils"; declare global { interface Window { @@ -96,8 +98,30 @@ export function GroupCallView({ const onJoin = async (ev: CustomEvent) => { const { audioInput, videoInput } = ev.detail .data as unknown as JoinCallData; - if (audioInput !== null) setAudioInput(audioInput); - if (videoInput !== null) setVideoInput(videoInput); + + if (audioInput !== null) { + const deviceId = await findDeviceByName(audioInput); + if (!deviceId) { + logger.warn("Unknown audio input: " + audioInput); + } else { + logger.debug( + `Found audio input ID ${deviceId} for name ${audioInput}` + ); + setAudioInput(deviceId); + } + } + + if (videoInput !== null) { + const deviceId = await findDeviceByName(videoInput); + if (!deviceId) { + logger.warn("Unknown video input: " + videoInput); + } else { + logger.debug( + `Found video input ID ${deviceId} for name ${videoInput}` + ); + setVideoInput(deviceId); + } + } await Promise.all([ groupCall.setMicrophoneMuted(audioInput === null), groupCall.setLocalVideoMuted(videoInput === null), From 17613837b674e210699ca2cad393ad09142f156c Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Sep 2022 13:19:46 +0100 Subject: [PATCH 2/4] Hold a user media stream open while we get devices As per comment. --- src/media-utils.ts | 49 ++++++++++++++++++++++++++++++++++++-- src/room/GroupCallView.tsx | 11 ++++++--- 2 files changed, 55 insertions(+), 5 deletions(-) diff --git a/src/media-utils.ts b/src/media-utils.ts index 0c01b5d..d00fc72 100644 --- a/src/media-utils.ts +++ b/src/media-utils.ts @@ -14,10 +14,55 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { logger } from "matrix-js-sdk/src/logger"; + +/** + * Finds a media device with label matching 'deviceName' + * @param deviceName The label of the device to look for + * @param devices The list of devices to search + * @returns A matching media device or undefined if no matching device was found + */ export async function findDeviceByName( - deviceName: string + deviceName: string, + devices: MediaDeviceInfo[] ): Promise { - const devices = await navigator.mediaDevices.enumerateDevices(); const deviceInfo = devices.find((d) => d.label === deviceName); return deviceInfo?.deviceId; } + +/** + * Gets the available audio input/output and video input devices + * from the browser: a wrapper around mediaDevices.enumerateDevices() + * that requests a stream and holds it while calling enumerateDevices(). + * This is because some browsers (Firefox) only return device labels when + * the app has an active user media stream. In Chrome, this will get a + * stream from the default camera which can mean, for example, that the + * light for the FaceTime camera turns on briefly even if you selected + * another camera. Once the Permissions API + * (https://developer.mozilla.org/en-US/docs/Web/API/Permissions_API) + * is ready for primetime, this should allow us to avoid this. + * + * @return The available media devices + */ +export async function getDevices(): Promise { + let stream; + try { + stream = await navigator.mediaDevices.getUserMedia({ + audio: true, + video: true, + }); + } catch (e) { + logger.info("Couldn't get media stream for enumerateDevices: failing"); + throw e; + } + + try { + return await navigator.mediaDevices.enumerateDevices(); + } catch (error) { + logger.warn("Unable to refresh WebRTC Devices: ", error); + } finally { + for (const track of stream.getTracks()) { + track.stop(); + } + } +} diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 0598e4d..be436c5 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -32,7 +32,7 @@ import { useRoomAvatar } from "./useRoomAvatar"; import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { useLocationNavigation } from "../useLocationNavigation"; import { useMediaHandler } from "../settings/useMediaHandler"; -import { findDeviceByName } from "../media-utils"; +import { findDeviceByName, getDevices } from "../media-utils"; declare global { interface Window { @@ -96,11 +96,16 @@ export function GroupCallView({ if (widget && preload) { // In preload mode, wait for a join action before entering const onJoin = async (ev: CustomEvent) => { + // Get the available devices so we can match the selected device + // to its ID. This involves getting a media stream (see docs on + // the function) so we only do it once and re-use the result. + const devices = await getDevices(); + const { audioInput, videoInput } = ev.detail .data as unknown as JoinCallData; if (audioInput !== null) { - const deviceId = await findDeviceByName(audioInput); + const deviceId = await findDeviceByName(audioInput, devices); if (!deviceId) { logger.warn("Unknown audio input: " + audioInput); } else { @@ -112,7 +117,7 @@ export function GroupCallView({ } if (videoInput !== null) { - const deviceId = await findDeviceByName(videoInput); + const deviceId = await findDeviceByName(videoInput, devices); if (!deviceId) { logger.warn("Unknown video input: " + videoInput); } else { From 77da0c912f4ea373bc306a2853667948db1a25f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Sep 2022 17:07:10 +0100 Subject: [PATCH 3/4] Match device type too Because lots of audio & video inputs have the same name --- src/media-utils.ts | 5 ++++- src/room/GroupCallView.tsx | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/media-utils.ts b/src/media-utils.ts index d00fc72..55650eb 100644 --- a/src/media-utils.ts +++ b/src/media-utils.ts @@ -24,9 +24,12 @@ import { logger } from "matrix-js-sdk/src/logger"; */ export async function findDeviceByName( deviceName: string, + kind: MediaDeviceKind, devices: MediaDeviceInfo[] ): Promise { - const deviceInfo = devices.find((d) => d.label === deviceName); + const deviceInfo = devices.find( + (d) => d.kind === kind && d.label === deviceName + ); return deviceInfo?.deviceId; } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index be436c5..b777444 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -105,7 +105,11 @@ export function GroupCallView({ .data as unknown as JoinCallData; if (audioInput !== null) { - const deviceId = await findDeviceByName(audioInput, devices); + const deviceId = await findDeviceByName( + audioInput, + "audioinput", + devices + ); if (!deviceId) { logger.warn("Unknown audio input: " + audioInput); } else { @@ -117,7 +121,11 @@ export function GroupCallView({ } if (videoInput !== null) { - const deviceId = await findDeviceByName(videoInput, devices); + const deviceId = await findDeviceByName( + videoInput, + "videoinput", + devices + ); if (!deviceId) { logger.warn("Unknown video input: " + videoInput); } else { From f808c561210e2dca1af64b3ae02eab2910b93bc1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 29 Sep 2022 17:08:48 +0100 Subject: [PATCH 4/4] Type --- src/media-utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/media-utils.ts b/src/media-utils.ts index 55650eb..e0841cc 100644 --- a/src/media-utils.ts +++ b/src/media-utils.ts @@ -48,7 +48,7 @@ export async function findDeviceByName( * @return The available media devices */ export async function getDevices(): Promise { - let stream; + let stream: MediaStream; try { stream = await navigator.mediaDevices.getUserMedia({ audio: true,