From 58e505cd38bf3dcc53043e9cf477d29f2e4128ce Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 4 Nov 2022 18:10:53 +0000 Subject: [PATCH 1/8] Add aria-describedby associations --- src/input/Input.tsx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/input/Input.tsx b/src/input/Input.tsx index cd7603f..047a7a6 100644 --- a/src/input/Input.tsx +++ b/src/input/Input.tsx @@ -122,6 +122,7 @@ export const InputField = forwardRef< type={type} checked={checked} disabled={disabled} + aria-describedby={description ? id + "-desc" : undefined} {...rest} /> )} @@ -135,7 +136,11 @@ export const InputField = forwardRef< {label} {suffix && {suffix}} - {description &&

{description}

} + {description && ( +

+ {description} +

+ )} ); } From 3cac74df245d729398806d522b533494d99ddd05 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 7 Nov 2022 12:28:54 +0000 Subject: [PATCH 2/8] Add aria-describedBy to textarea and use useID --- src/input/Input.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/input/Input.tsx b/src/input/Input.tsx index 047a7a6..738a151 100644 --- a/src/input/Input.tsx +++ b/src/input/Input.tsx @@ -16,6 +16,7 @@ limitations under the License. import React, { ChangeEvent, FC, forwardRef, ReactNode } from "react"; import classNames from "classnames"; +import { useId } from "@react-aria/utils"; import styles from "./Input.module.css"; import { ReactComponent as CheckIcon } from "../icons/Check.svg"; @@ -96,6 +97,8 @@ export const InputField = forwardRef< }, ref ) => { + const descriptionId = useId(id + "-desc"); + return ( } disabled={disabled} + aria-describedby={descriptionId} {...rest} /> ) : ( @@ -122,7 +126,7 @@ export const InputField = forwardRef< type={type} checked={checked} disabled={disabled} - aria-describedby={description ? id + "-desc" : undefined} + aria-describedby={descriptionId} {...rest} /> )} @@ -137,7 +141,7 @@ export const InputField = forwardRef< {suffix && {suffix}} {description && ( -

+

{description}

)} From d5a5ce9860459e6928a96b044a5dc2446db4e47d Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 7 Nov 2022 14:05:58 +0000 Subject: [PATCH 3/8] Don't pass potentially undefined 'desc' to useId Also use the useId that comes with React 18. --- src/input/Input.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/input/Input.tsx b/src/input/Input.tsx index 738a151..acec7d3 100644 --- a/src/input/Input.tsx +++ b/src/input/Input.tsx @@ -14,9 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ -import React, { ChangeEvent, FC, forwardRef, ReactNode } from "react"; +import React, { ChangeEvent, FC, forwardRef, ReactNode, useId } from "react"; import classNames from "classnames"; -import { useId } from "@react-aria/utils"; import styles from "./Input.module.css"; import { ReactComponent as CheckIcon } from "../icons/Check.svg"; @@ -97,7 +96,7 @@ export const InputField = forwardRef< }, ref ) => { - const descriptionId = useId(id + "-desc"); + const descriptionId = useId(); return ( Date: Mon, 7 Nov 2022 11:41:49 -0500 Subject: [PATCH 4/8] Add logs to debug missing spatial audio And potentially fix it, by recreating the source node when the stream changes. --- src/video-grid/VideoTileContainer.tsx | 4 +- src/video-grid/useMediaStream.ts | 109 ++++++++++++++------------ 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index cb93dbd..fe3cb23 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -71,8 +71,8 @@ export function VideoTileContainer({ stream ?? null, audioContext, audioDestination, - isLocal, - localVolume + localVolume, + isLocal ); const { modalState: videoTileSettingsModalState, diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts index 61c6524..d1879bd 100644 --- a/src/video-grid/useMediaStream.ts +++ b/src/video-grid/useMediaStream.ts @@ -20,6 +20,7 @@ import { acquireContext, releaseContext, } from "matrix-js-sdk/src/webrtc/audioContext"; +import { logger } from "matrix-js-sdk/src/logger"; import { useSpatialAudio } from "../settings/useSetting"; import { useEventTarget } from "../useEvents"; @@ -213,10 +214,10 @@ export const useSpatialMediaStream = ( stream: MediaStream | null, audioContext: AudioContext, audioDestination: AudioNode, - mute = false, - localVolume?: number + localVolume: number, + mute = false ): [RefObject, RefObject] => { - const tileRef = useRef(); + const tileRef = useRef(null); const [spatialAudio] = useSpatialAudio(); // We always handle audio separately form the video element const mediaRef = useMediaStream(stream, null, true); @@ -227,53 +228,63 @@ export const useSpatialMediaStream = ( const sourceRef = useRef(); useEffect(() => { - if (spatialAudio && tileRef.current && !mute && audioTrackCount > 0) { - if (!pannerNodeRef.current) { - pannerNodeRef.current = new PannerNode(audioContext, { - panningModel: "HRTF", - refDistance: 3, - }); + if (spatialAudio) { + if (tileRef.current && !mute && audioTrackCount > 0) { + logger.debug(`Rendering spatial audio for ${stream!.id}`); + + if (!pannerNodeRef.current) { + pannerNodeRef.current = new PannerNode(audioContext, { + panningModel: "HRTF", + refDistance: 3, + }); + } + if (!gainNodeRef.current) { + gainNodeRef.current = new GainNode(audioContext, { + gain: localVolume, + }); + } + if (!sourceRef.current || sourceRef.current.mediaStream !== stream!) { + sourceRef.current = audioContext.createMediaStreamSource(stream!); + } + + const tile = tileRef.current; + const source = sourceRef.current; + const gainNode = gainNodeRef.current; + const pannerNode = pannerNodeRef.current; + + const updatePosition = () => { + const bounds = tile.getBoundingClientRect(); + const windowSize = Math.max(window.innerWidth, window.innerHeight); + // Position the source relative to its placement in the window + pannerNodeRef.current!.positionX.value = + (bounds.x + bounds.width / 2) / windowSize - 0.5; + pannerNodeRef.current!.positionY.value = + (bounds.y + bounds.height / 2) / windowSize - 0.5; + // Put the source in front of the listener + pannerNodeRef.current!.positionZ.value = -2; + }; + + gainNode.gain.value = localVolume; + updatePosition(); + source.connect(gainNode).connect(pannerNode).connect(audioDestination); + // HACK: We abuse the CSS transitionrun event to detect when the tile + // moves, because useMeasure, IntersectionObserver, etc. all have no + // ability to track changes in the CSS transform property + tile.addEventListener("transitionrun", updatePosition); + + return () => { + tile.removeEventListener("transitionrun", updatePosition); + source.disconnect(); + gainNode.disconnect(); + pannerNode.disconnect(); + }; + } else if (stream) { + logger.debug( + `Not rendering spatial audio for ${stream.id} (tile ref ${Boolean( + tileRef.current + )}, mute ${mute}, track count ${audioTrackCount})` + ); } - if (!gainNodeRef.current) { - gainNodeRef.current = new GainNode(audioContext, { - gain: localVolume, - }); - } - if (!sourceRef.current) { - sourceRef.current = audioContext.createMediaStreamSource(stream!); - } - - const tile = tileRef.current; - const source = sourceRef.current; - const gainNode = gainNodeRef.current; - const pannerNode = pannerNodeRef.current; - - const updatePosition = () => { - const bounds = tile.getBoundingClientRect(); - const windowSize = Math.max(window.innerWidth, window.innerHeight); - // Position the source relative to its placement in the window - pannerNodeRef.current!.positionX.value = - (bounds.x + bounds.width / 2) / windowSize - 0.5; - pannerNodeRef.current!.positionY.value = - (bounds.y + bounds.height / 2) / windowSize - 0.5; - // Put the source in front of the listener - pannerNodeRef.current!.positionZ.value = -2; - }; - - gainNode.gain.value = localVolume; - updatePosition(); - source.connect(gainNode).connect(pannerNode).connect(audioDestination); - // HACK: We abuse the CSS transitionrun event to detect when the tile - // moves, because useMeasure, IntersectionObserver, etc. all have no - // ability to track changes in the CSS transform property - tile.addEventListener("transitionrun", updatePosition); - - return () => { - tile.removeEventListener("transitionrun", updatePosition); - source.disconnect(); - gainNode.disconnect(); - pannerNode.disconnect(); - }; } }, [ stream, From 70344fd40f3c13ea49053e83b76833fb6d5b0e66 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 7 Nov 2022 11:50:05 -0500 Subject: [PATCH 5/8] Disable spatial audio for the maximized speaker --- src/video-grid/VideoTileContainer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index cb93dbd..28fa047 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -71,7 +71,7 @@ export function VideoTileContainer({ stream ?? null, audioContext, audioDestination, - isLocal, + isLocal || maximised, localVolume ); const { From e1336254051feb99f3f12306186af2be4d7965aa Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 7 Nov 2022 17:37:40 +0000 Subject: [PATCH 6/8] Don't check out group-call branch in build script This build script might change more soon (we shouldn't really need to checkout & link the js-sdk at all) but for now let's just switch the branch now group-call is merged. --- scripts/dockerbuild.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/dockerbuild.sh b/scripts/dockerbuild.sh index 7dce8f6..cda60c2 100755 --- a/scripts/dockerbuild.sh +++ b/scripts/dockerbuild.sh @@ -7,7 +7,6 @@ export VITE_PRODUCT_NAME="Element Call" git clone https://github.com/matrix-org/matrix-js-sdk.git cd matrix-js-sdk -git checkout robertlong/group-call yarn install yarn run build yarn link From 5f84cb5790861930687686a3134a1c22459564d1 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 7 Nov 2022 14:03:28 -0500 Subject: [PATCH 7/8] Fix lint --- src/video-grid/VideoTileContainer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index afe3d64..0478bb4 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -72,7 +72,7 @@ export function VideoTileContainer({ audioContext, audioDestination, localVolume, - isLocal || maximised, + isLocal || maximised ); const { modalState: videoTileSettingsModalState, From ca6d75e384d88779b6ac86369eb60153d3ee51b5 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Tue, 8 Nov 2022 07:53:17 -0500 Subject: [PATCH 8/8] Improve the analytics opt-in copy --- public/locales/en-GB/app.json | 2 +- src/settings/SettingsModal.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index ef330e6..7d4ae7b 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -117,7 +117,7 @@ "This call already exists, would you like to join?": "This call already exists, would you like to join?", "This site is protected by ReCAPTCHA and the Google <2>Privacy Policy and <6>Terms of Service apply.<9>By clicking \"Register\", you agree to our <12>Terms and conditions": "This site is protected by ReCAPTCHA and the Google <2>Privacy Policy and <6>Terms of Service apply.<9>By clicking \"Register\", you agree to our <12>Terms and conditions", "This will make a speaker's audio seem as if it is coming from where their tile is positioned on screen. (Experimental feature: this may impact the stability of audio.)": "This will make a speaker's audio seem as if it is coming from where their tile is positioned on screen. (Experimental feature: this may impact the stability of audio.)", - "This will send anonymized data (such as the duration of a call and the number of participants) to the element call team to help us optimise the application based on how it is used.": "This will send anonymized data (such as the duration of a call and the number of participants) to the element call team to help us optimise the application based on how it is used.", + "This will send anonymised data (such as the duration of a call and the number of participants) to the Element Call team to help us optimise the application based on how it is used.": "This will send anonymised data (such as the duration of a call and the number of participants) to the Element Call team to help us optimise the application based on how it is used.", "Turn off camera": "Turn off camera", "Turn on camera": "Turn on camera", "Unmute microphone": "Unmute microphone", diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index cb7acb2..f895538 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -159,7 +159,7 @@ export const SettingsModal = (props: Props) => { type="checkbox" checked={optInAnalytics} description={t( - "This will send anonymized data (such as the duration of a call and the number of participants) to the element call team to help us optimise the application based on how it is used." + "This will send anonymised data (such as the duration of a call and the number of participants) to the Element Call team to help us optimise the application based on how it is used." )} onChange={(event: React.ChangeEvent) => setOptInAnalytics(event.target.checked)