Merge branch 'main' into ts_button

This commit is contained in:
Timo K 2022-07-02 21:42:15 +02:00
commit f1bd47be8c
31 changed files with 958 additions and 436 deletions

1
.env
View file

@ -22,6 +22,7 @@
# VITE_THEME_PRIMARY_CONTENT=#ffffff
# VITE_THEME_SECONDARY_CONTENT=#a9b2bc
# VITE_THEME_TERTIARY_CONTENT=#8e99a4
# VITE_THEME_TERTIARY_CONTENT_20=#8e99a433
# VITE_THEME_QUATERNARY_CONTENT=#6f7882
# VITE_THEME_QUINARY_CONTENT=#394049
# VITE_THEME_SYSTEM=#21262c

67
.github/ISSUE_TEMPLATE/bug.yml vendored Normal file
View file

@ -0,0 +1,67 @@
name: Bug report
description: Create a report to help us improve
labels: [T-Defect]
body:
- type: markdown
attributes:
value: |
Thanks for taking the time to fill out this bug report!
Please report security issues by email to security@matrix.org
- type: textarea
id: reproduction-steps
attributes:
label: Steps to reproduce
description: Please attach screenshots, videos or logs if you can.
placeholder: Tell us what you see!
value: |
1. Where are you starting? What can you see?
2. What do you click?
3. More steps…
validations:
required: true
- type: textarea
id: result
attributes:
label: Outcome
placeholder: Tell us what went wrong
value: |
#### What did you expect?
#### What happened instead?
validations:
required: true
- type: input
id: os
attributes:
label: Operating system
placeholder: Windows, macOS, Ubuntu, Android…
validations:
required: false
- type: input
id: browser
attributes:
label: Browser information
description: Which browser are you using? Which version?
placeholder: e.g. Chromium Version 92.0.4515.131
validations:
required: false
- type: input
id: webapp-url
attributes:
label: URL for webapp
description: Which URL are you using to access the webapp? If a private server, tell us what version of Element Call you are using.
placeholder: e.g. call.element.io
validations:
required: false
- type: dropdown
id: rageshake
attributes:
label: Will you send logs?
description: |
To send them, press the 'Submit Feedback' button and check 'Include Debug Logs'. Please link to this issue in the description field.
options:
- 'Yes'
- 'No'
validations:
required: true

5
.github/ISSUE_TEMPLATE/config.yml vendored Normal file
View file

@ -0,0 +1,5 @@
blank_issues_enabled: false
contact_links:
- name: Questions & support
url: https://matrix.to/#/#webrtc:matrix.org
about: Please ask and answer questions here.

36
.github/ISSUE_TEMPLATE/enhancement.yml vendored Normal file
View file

@ -0,0 +1,36 @@
name: Enhancement request
description: Do you have a suggestion or feature request?
labels: [T-Enhancement]
body:
- type: markdown
attributes:
value: |
Thank you for taking the time to propose a new feature or make a suggestion.
- type: textarea
id: usecase
attributes:
label: Your use case
description: What would you like to be able to do? Please feel welcome to include screenshots or mock ups.
placeholder: Tell us what you would like to do!
value: |
#### What would you like to do?
#### Why would you like to do it?
#### How would you like to achieve it?
validations:
required: true
- type: textarea
id: alternative
attributes:
label: Have you considered any alternatives?
placeholder: A clear and concise description of any alternative solutions or features you've considered.
validations:
required: false
- type: textarea
id: additional-context
attributes:
label: Additional context
placeholder: Is there anything else you'd like to add?
validations:
required: false

View file

@ -33,11 +33,12 @@
"@sentry/react": "^6.13.3",
"@sentry/tracing": "^6.13.3",
"@types/grecaptcha": "^3.0.4",
"@types/sdp-transform": "^2.4.5",
"@use-gesture/react": "^10.2.11",
"classnames": "^2.3.1",
"color-hash": "^2.0.1",
"events": "^3.3.0",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#404f8e130e44a78b0159c55902df1b129b3816d1",
"matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f9672cf3076bf5e1868c92ee94c780040b0c6fb1",
"mermaid": "^8.13.8",
"normalize.css": "^8.0.1",
"pako": "^2.0.4",
@ -50,6 +51,7 @@
"react-router-dom": "^5.2.0",
"react-use-clipboard": "^1.0.7",
"react-use-measure": "^2.1.1",
"sdp-transform": "^2.14.1",
"unique-names-generator": "^4.6.0"
},
"devDependencies": {

View file

@ -21,4 +21,10 @@ declare global {
// TODO: https://gitlab.matrix.org/matrix-org/olm/-/issues/10
OLM_OPTIONS: Record<string, string>;
}
// TypeScript doesn't know about the experimental setSinkId method, so we
// declare it ourselves
interface MediaElement extends HTMLMediaElement {
setSinkId: (id: string) => void;
}
}

View file

@ -77,9 +77,23 @@ export function RoomHeaderInfo({ roomName, avatarUrl }) {
);
}
export function RoomSetupHeaderInfo({ roomName, avatarUrl, ...rest }) {
export function RoomSetupHeaderInfo({
roomName,
avatarUrl,
isEmbedded,
...rest
}) {
const ref = useRef();
const { buttonProps } = useButton(rest, ref);
if (isEmbedded) {
return (
<div ref={ref}>
<RoomHeaderInfo roomName={roomName} avatarUrl={avatarUrl} />
</div>
);
}
return (
<button className={styles.backButton} ref={ref} {...buttonProps}>
<ArrowLeftIcon width={16} height={16} />

View file

@ -33,6 +33,7 @@ limitations under the License.
--primary-content: #ffffff;
--secondary-content: #a9b2bc;
--tertiary-content: #8e99a4;
--tertiary-content-20: #8e99a433;
--quaternary-content: #6f7882;
--quinary-content: #394049;
--system: #21262c;

View file

@ -61,6 +61,10 @@ if (import.meta.env.VITE_CUSTOM_THEME) {
"--tertiary-content",
import.meta.env.VITE_THEME_TERTIARY_CONTENT as string
);
style.setProperty(
"--tertiary-content-20",
import.meta.env.VITE_THEME_TERTIARY_CONTENT_20 as string
);
style.setProperty(
"--quaternary-content",
import.meta.env.VITE_THEME_QUATERNARY_CONTENT as string

View file

@ -1,9 +1,10 @@
import Olm from "@matrix-org/olm";
import olmWasmPath from "@matrix-org/olm/olm.wasm?url";
import { IndexedDBStore } from "matrix-js-sdk/src/store/indexeddb";
import { WebStorageSessionStore } from "matrix-js-sdk/src/store/session/webstorage";
import { MemoryStore } from "matrix-js-sdk/src/store/memory";
import { IndexedDBCryptoStore } from "matrix-js-sdk/src/crypto/store/indexeddb-crypto-store";
import { LocalStorageCryptoStore } from "matrix-js-sdk/src/crypto/store/localStorage-crypto-store";
import { MemoryCryptoStore } from "matrix-js-sdk/src/crypto/store/memory-crypto-store";
import { createClient, MatrixClient } from "matrix-js-sdk/src/matrix";
import { ICreateClientOpts } from "matrix-js-sdk/src/matrix";
import { ClientEvent } from "matrix-js-sdk/src/client";
@ -59,14 +60,12 @@ export async function initClient(
if (indexedDB && localStorage && !import.meta.env.DEV) {
storeOpts.store = new IndexedDBStore({
indexedDB: window.indexedDB,
localStorage: window.localStorage,
localStorage,
dbName: "element-call-sync",
workerFactory: () => new IndexedDBWorker(),
});
}
if (localStorage) {
storeOpts.sessionStore = new WebStorageSessionStore(localStorage);
} else if (localStorage) {
storeOpts.store = new MemoryStore({ localStorage });
}
if (indexedDB) {
@ -74,6 +73,10 @@ export async function initClient(
indexedDB,
"matrix-js-sdk:crypto"
);
} else if (localStorage) {
storeOpts.cryptoStore = new LocalStorageCryptoStore(localStorage);
} else {
storeOpts.cryptoStore = new MemoryCryptoStore();
}
const client = createClient({

View file

@ -15,6 +15,8 @@ limitations under the License.
*/
import React, { useCallback, useEffect, useState } from "react";
import { MatrixClient } from "matrix-js-sdk";
import { Button } from "../button";
import { useProfile } from "./useProfile";
import { FieldRow, InputField, ErrorMessage } from "../input/Input";
@ -22,7 +24,12 @@ import { Modal, ModalContent } from "../Modal";
import { AvatarInputField } from "../input/AvatarInputField";
import styles from "./ProfileModal.module.css";
export function ProfileModal({ client, ...rest }) {
interface Props {
client: MatrixClient;
onClose: () => {};
[rest: string]: unknown;
}
export function ProfileModal({ client, ...rest }: Props) {
const { onClose } = rest;
const {
success,
@ -50,13 +57,20 @@ export function ProfileModal({ client, ...rest }) {
(e) => {
e.preventDefault();
const data = new FormData(e.target);
const displayName = data.get("displayName");
const avatar = data.get("avatar");
const displayNameDataEntry = data.get("displayName");
const avatar: File | string = data.get("avatar");
const avatarSize =
typeof avatar == "string" ? avatar.length : avatar.size;
const displayName =
typeof displayNameDataEntry == "string"
? displayNameDataEntry
: displayNameDataEntry.name;
saveProfile({
displayName,
avatar: avatar && avatar.size > 0 ? avatar : undefined,
removeAvatar: removeAvatar && (!avatar || avatar.size === 0),
avatar: avatar && avatarSize > 0 ? avatar : undefined,
removeAvatar: removeAvatar && (!avatar || avatarSize === 0),
});
},
[saveProfile, removeAvatar]

View file

@ -14,11 +14,33 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import { MatrixClient } from "matrix-js-sdk/src/client";
import { MatrixEvent } from "matrix-js-sdk/src/models/event";
import { User, UserEvent } from "matrix-js-sdk/src/models/user";
import { FileType } from "matrix-js-sdk/src/http-api";
import { useState, useCallback, useEffect } from "react";
export function useProfile(client) {
interface ProfileLoadState {
success?: boolean;
loading?: boolean;
displayName: string;
avatarUrl: string;
error?: Error;
}
type ProfileSaveCallback = ({
displayName,
avatar,
removeAvatar,
}: {
displayName: string;
avatar: FileType;
removeAvatar: boolean;
}) => Promise<void>;
export function useProfile(client: MatrixClient) {
const [{ loading, displayName, avatarUrl, error, success }, setState] =
useState(() => {
useState<ProfileLoadState>(() => {
const user = client?.getUser(client.getUserId());
return {
@ -31,7 +53,10 @@ export function useProfile(client) {
});
useEffect(() => {
const onChangeUser = (_event, { displayName, avatarUrl }) => {
const onChangeUser = (
_event: MatrixEvent,
{ displayName, avatarUrl }: User
) => {
setState({
success: false,
loading: false,
@ -41,24 +66,24 @@ export function useProfile(client) {
});
};
let user;
let user: User;
if (client) {
const userId = client.getUserId();
user = client.getUser(userId);
user.on("User.displayName", onChangeUser);
user.on("User.avatarUrl", onChangeUser);
user.on(UserEvent.DisplayName, onChangeUser);
user.on(UserEvent.AvatarUrl, onChangeUser);
}
return () => {
if (user) {
user.removeListener("User.displayName", onChangeUser);
user.removeListener("User.avatarUrl", onChangeUser);
user.removeListener(UserEvent.DisplayName, onChangeUser);
user.removeListener(UserEvent.AvatarUrl, onChangeUser);
}
};
}, [client]);
const saveProfile = useCallback(
const saveProfile = useCallback<ProfileSaveCallback>(
async ({ displayName, avatar, removeAvatar }) => {
if (client) {
setState((prev) => ({
@ -71,7 +96,7 @@ export function useProfile(client) {
try {
await client.setDisplayName(displayName);
let mxcAvatarUrl;
let mxcAvatarUrl: string;
if (removeAvatar) {
await client.setAvatarUrl("");
@ -87,11 +112,11 @@ export function useProfile(client) {
loading: false,
success: true,
}));
} catch (error) {
} catch (error: unknown) {
setState((prev) => ({
...prev,
loading: false,
error,
error: error instanceof Error ? error : Error(error as string),
success: false,
}));
}
@ -102,5 +127,12 @@ export function useProfile(client) {
[client]
);
return { loading, error, displayName, avatarUrl, saveProfile, success };
return {
loading,
error,
displayName,
avatarUrl,
saveProfile,
success,
};
}

View file

@ -30,6 +30,7 @@ import { useLocationNavigation } from "../useLocationNavigation";
export function GroupCallView({
client,
isPasswordlessUser,
isEmbedded,
roomId,
groupCall,
}) {
@ -92,6 +93,7 @@ export function GroupCallView({
participants={participants}
userMediaFeeds={userMediaFeeds}
onLeave={onLeave}
isEmbedded={isEmbedded}
/>
);
} else {

View file

@ -42,6 +42,7 @@ import { usePreventScroll } from "@react-aria/overlays";
import { useMediaHandler } from "../settings/useMediaHandler";
import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
import { useAudioContext } from "../video-grid/useMediaStream";
const canScreenshare = "getDisplayMedia" in navigator.mediaDevices;
// There is currently a bug in Safari our our code with cloning and sending MediaStreams
@ -70,12 +71,10 @@ export function InCallView({
usePreventScroll();
const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0);
const [audioContext, audioDestination, audioRef] = useAudioContext();
const { audioOutput } = useMediaHandler();
const [showInspector] = useShowInspector();
const audioContext = useRef();
if (!audioContext.current) audioContext.current = new AudioContext();
const { modalState: feedbackModalState, modalProps: feedbackModalProps } =
useModalTriggerState();
@ -139,6 +138,7 @@ export function InCallView({
return (
<div className={styles.inRoom}>
<audio ref={audioRef} />
<Header>
<LeftNav>
<RoomHeaderInfo roomName={roomName} avatarUrl={avatarUrl} />
@ -165,7 +165,8 @@ export function InCallView({
getAvatar={renderAvatar}
showName={items.length > 2 || item.focused}
audioOutputDevice={audioOutput}
audioContext={audioContext.current}
audioContext={audioContext}
audioDestination={audioDestination}
disableSpeakingIndicator={items.length < 3}
{...rest}
/>

View file

@ -17,6 +17,12 @@
cursor: unset;
}
.networkWaiting {
background-color: var(--tertiary-content);
border-color: var(--tertiary-content);
cursor: unset;
}
.error {
background-color: var(--alert);
border-color: var(--alert);

View file

@ -14,15 +14,17 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React, { useCallback, useEffect, useState, createRef } from "react";
import React, { useCallback, useState, createRef } from "react";
import classNames from "classnames";
import { useSpring, animated } from "@react-spring/web";
import styles from "./PTTButton.module.css";
import { ReactComponent as MicIcon } from "../icons/Mic.svg";
import { useEventTarget } from "../useEvents";
import { Avatar } from "../Avatar";
interface Props {
enabled: boolean;
showTalkOverError: boolean;
activeSpeakerUserId: string;
activeSpeakerDisplayName: string;
@ -32,15 +34,13 @@ interface Props {
size: number;
startTalking: () => void;
stopTalking: () => void;
}
interface State {
isHeld: boolean;
// If the button is being pressed by touch, the ID of that touch
activeTouchID: number | null;
networkWaiting: boolean;
enqueueNetworkWaiting: (value: boolean, delay: number) => void;
setNetworkWaiting: (value: boolean) => void;
}
export const PTTButton: React.FC<Props> = ({
enabled,
showTalkOverError,
activeSpeakerUserId,
activeSpeakerDisplayName,
@ -50,89 +50,110 @@ export const PTTButton: React.FC<Props> = ({
size,
startTalking,
stopTalking,
networkWaiting,
enqueueNetworkWaiting,
setNetworkWaiting,
}) => {
const buttonRef = createRef<HTMLButtonElement>();
const [{ isHeld, activeTouchID }, setState] = useState<State>({
isHeld: false,
activeTouchID: null,
});
const onWindowMouseUp = useCallback(
(e) => {
if (isHeld) stopTalking();
setState({ isHeld: false, activeTouchID: null });
},
[isHeld, setState, stopTalking]
);
const [activeTouchId, setActiveTouchId] = useState<number | null>(null);
const onWindowTouchEnd = useCallback(
(e: TouchEvent) => {
// ignore any ended touches that weren't the one pressing the
// button (bafflingly the TouchList isn't an iterable so we
// have to do this a really old-school way).
let touchFound = false;
for (let i = 0; i < e.changedTouches.length; ++i) {
if (e.changedTouches.item(i).identifier === activeTouchID) {
touchFound = true;
break;
}
}
if (!touchFound) return;
e.preventDefault();
if (isHeld) stopTalking();
setState({ isHeld: false, activeTouchID: null });
},
[isHeld, activeTouchID, setState, stopTalking]
);
const hold = useCallback(() => {
// This update is delayed so the user only sees it if latency is significant
enqueueNetworkWaiting(true, 100);
startTalking();
}, [enqueueNetworkWaiting, startTalking]);
const unhold = useCallback(() => {
setNetworkWaiting(false);
stopTalking();
}, [setNetworkWaiting, stopTalking]);
const onButtonMouseDown = useCallback(
(e: React.MouseEvent<HTMLButtonElement>) => {
e.preventDefault();
setState({ isHeld: true, activeTouchID: null });
startTalking();
hold();
},
[setState, startTalking]
[hold]
);
const onButtonTouchStart = useCallback(
(e: TouchEvent) => {
e.preventDefault();
// These listeners go on the window so even if the user's cursor / finger
// leaves the button while holding it, the button stays pushed until
// they stop clicking / tapping.
useEventTarget(window, "mouseup", unhold);
useEventTarget(
window,
"touchend",
useCallback(
(e: TouchEvent) => {
// ignore any ended touches that weren't the one pressing the
// button (bafflingly the TouchList isn't an iterable so we
// have to do this a really old-school way).
let touchFound = false;
for (let i = 0; i < e.changedTouches.length; ++i) {
if (e.changedTouches.item(i).identifier === activeTouchId) {
touchFound = true;
break;
}
}
if (!touchFound) return;
if (isHeld) return;
setState({
isHeld: true,
activeTouchID: e.changedTouches.item(0).identifier,
});
startTalking();
},
[isHeld, setState, startTalking]
e.preventDefault();
unhold();
setActiveTouchId(null);
},
[unhold, activeTouchId, setActiveTouchId]
)
);
useEffect(() => {
const currentButtonElement = buttonRef.current;
// This is a native DOM listener too because we want to preventDefault in it
// to stop also getting a click event, so we need it to be non-passive.
useEventTarget(
buttonRef.current,
"touchstart",
useCallback(
(e: TouchEvent) => {
e.preventDefault();
// These listeners go on the window so even if the user's cursor / finger
// leaves the button while holding it, the button stays pushed until
// they stop clicking / tapping.
window.addEventListener("mouseup", onWindowMouseUp);
window.addEventListener("touchend", onWindowTouchEnd);
// This is a native DOM listener too because we want to preventDefault in it
// to stop also getting a click event, so we need it to be non-passive.
currentButtonElement.addEventListener("touchstart", onButtonTouchStart, {
passive: false,
});
hold();
setActiveTouchId(e.changedTouches.item(0).identifier);
},
[hold, setActiveTouchId]
),
{ passive: false }
);
return () => {
window.removeEventListener("mouseup", onWindowMouseUp);
window.removeEventListener("touchend", onWindowTouchEnd);
currentButtonElement.removeEventListener(
"touchstart",
onButtonTouchStart
);
};
}, [onWindowMouseUp, onWindowTouchEnd, onButtonTouchStart, buttonRef]);
useEventTarget(
window,
"keydown",
useCallback(
(e: KeyboardEvent) => {
if (e.code === "Space") {
if (!enabled) return;
e.preventDefault();
hold();
}
},
[enabled, hold]
)
);
useEventTarget(
window,
"keyup",
useCallback(
(e: KeyboardEvent) => {
if (e.code === "Space") {
e.preventDefault();
unhold();
}
},
[unhold]
)
);
// TODO: We will need to disable this for a global PTT hotkey to work
useEventTarget(window, "blur", unhold);
const { shadow } = useSpring({
shadow: (Math.max(activeSpeakerVolume, -70) + 70) * 0.6,
@ -143,12 +164,15 @@ export const PTTButton: React.FC<Props> = ({
});
const shadowColor = showTalkOverError
? "var(--alert-20)"
: networkWaiting
? "var(--tertiary-content-20)"
: "var(--accent-20)";
return (
<animated.button
className={classNames(styles.pttButton, {
[styles.talking]: activeSpeakerUserId,
[styles.networkWaiting]: networkWaiting,
[styles.error]: showTalkOverError,
})}
style={{

View file

@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import React from "react";
import React, { useEffect } from "react";
import useMeasure from "react-use-measure";
import { ResizeObserver } from "@juggle/resize-observer";
import { GroupCall, MatrixClient, RoomMember } from "matrix-js-sdk";
import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
import { useDelayedState } from "../useDelayedState";
import { useModalTriggerState } from "../Modal";
import { InviteModal } from "./InviteModal";
import { HangupButton, InviteButton } from "../button";
@ -39,6 +40,7 @@ import { GroupCallInspector } from "./GroupCallInspector";
import { OverflowMenu } from "./OverflowMenu";
function getPromptText(
networkWaiting: boolean,
showTalkOverError: boolean,
pttButtonHeld: boolean,
activeSpeakerIsLocalUser: boolean,
@ -47,10 +49,14 @@ function getPromptText(
activeSpeakerDisplayName: string,
connected: boolean
): string {
if (!connected) return "Connection Lost";
if (!connected) return "Connection lost";
const isTouchScreen = Boolean(window.ontouchstart !== undefined);
if (networkWaiting) {
return "Waiting for network";
}
if (showTalkOverError) {
return "You can't talk at the same time";
}
@ -87,6 +93,7 @@ interface Props {
participants: RoomMember[];
userMediaFeeds: CallFeed[];
onLeave: () => void;
isEmbedded: boolean;
}
export const PTTCallView: React.FC<Props> = ({
@ -98,6 +105,7 @@ export const PTTCallView: React.FC<Props> = ({
participants,
userMediaFeeds,
onLeave,
isEmbedded,
}) => {
const { modalState: inviteModalState, modalProps: inviteModalProps } =
useModalTriggerState();
@ -128,18 +136,15 @@ export const PTTCallView: React.FC<Props> = ({
stopTalking,
transmitBlocked,
connected,
} = usePTT(
client,
groupCall,
userMediaFeeds,
playClip,
!feedbackModalState.isOpen
);
} = usePTT(client, groupCall, userMediaFeeds, playClip);
const [talkingExpected, enqueueTalkingExpected, setTalkingExpected] =
useDelayedState(false);
const showTalkOverError = pttButtonHeld && transmitBlocked;
const networkWaiting =
talkingExpected && !activeSpeakerUserId && !showTalkOverError;
const activeSpeakerIsLocalUser =
activeSpeakerUserId && client.getUserId() === activeSpeakerUserId;
const activeSpeakerIsLocalUser = activeSpeakerUserId === client.getUserId();
const activeSpeakerUser = activeSpeakerUserId
? client.getUser(activeSpeakerUserId)
: null;
@ -148,6 +153,10 @@ export const PTTCallView: React.FC<Props> = ({
? activeSpeakerUser.displayName
: "";
useEffect(() => {
setTalkingExpected(activeSpeakerIsLocalUser);
}, [activeSpeakerIsLocalUser, setTalkingExpected]);
return (
<div className={styles.pttCallView} ref={containerRef}>
<PTTClips
@ -169,6 +178,7 @@ export const PTTCallView: React.FC<Props> = ({
roomName={roomName}
avatarUrl={avatarUrl}
onPress={onLeave}
isEmbedded={isEmbedded}
/>
</LeftNav>
<RightNav />
@ -196,7 +206,7 @@ export const PTTCallView: React.FC<Props> = ({
feedbackModalState={feedbackModalState}
feedbackModalProps={feedbackModalProps}
/>
<HangupButton onPress={onLeave} />
{!isEmbedded && <HangupButton onPress={onLeave} />}
<InviteButton onPress={() => inviteModalState.open()} />
</div>
@ -217,6 +227,7 @@ export const PTTCallView: React.FC<Props> = ({
<div className={styles.talkingInfo} />
)}
<PTTButton
enabled={!feedbackModalState.isOpen}
showTalkOverError={showTalkOverError}
activeSpeakerUserId={activeSpeakerUserId}
activeSpeakerDisplayName={activeSpeakerDisplayName}
@ -226,9 +237,13 @@ export const PTTCallView: React.FC<Props> = ({
size={pttButtonSize}
startTalking={startTalking}
stopTalking={stopTalking}
networkWaiting={networkWaiting}
enqueueNetworkWaiting={enqueueTalkingExpected}
setNetworkWaiting={setTalkingExpected}
/>
<p className={styles.actionTip}>
{getPromptText(
networkWaiting,
showTalkOverError,
pttButtonHeld,
activeSpeakerIsLocalUser,

View file

@ -29,9 +29,9 @@ export function RoomPage() {
const { roomId: maybeRoomId } = useParams();
const { hash, search } = useLocation();
const [viaServers] = useMemo(() => {
const [viaServers, isEmbedded] = useMemo(() => {
const params = new URLSearchParams(search);
return [params.getAll("via")];
return [params.getAll("via"), params.has("embed")];
}, [search]);
const roomId = (maybeRoomId || hash || "").toLowerCase();
@ -56,6 +56,7 @@ export function RoomPage() {
roomId={roomId}
groupCall={groupCall}
isPasswordlessUser={isPasswordlessUser}
isEmbedded={isEmbedded}
/>
)}
</GroupCallLoader>

View file

@ -80,8 +80,7 @@ export const usePTT = (
client: MatrixClient,
groupCall: GroupCall,
userMediaFeeds: CallFeed[],
playClip: PlayClipFunction,
enablePTTButton: boolean
playClip: PlayClipFunction
): PTTState => {
// Used to serialise all the mute calls so they don't race. It has
// its own state as its always set separately from anything else.
@ -258,59 +257,6 @@ export const usePTT = (
[setConnected]
);
useEffect(() => {
function onKeyDown(event: KeyboardEvent): void {
if (event.code === "Space") {
if (!enablePTTButton) return;
event.preventDefault();
if (pttButtonHeld) return;
startTalking();
}
}
function onKeyUp(event: KeyboardEvent): void {
if (event.code === "Space") {
event.preventDefault();
stopTalking();
}
}
function onBlur(): void {
// TODO: We will need to disable this for a global PTT hotkey to work
if (!groupCall.isMicrophoneMuted()) {
setMicMuteWrapper(true);
}
setState((prevState) => ({ ...prevState, pttButtonHeld: false }));
}
window.addEventListener("keydown", onKeyDown);
window.addEventListener("keyup", onKeyUp);
window.addEventListener("blur", onBlur);
return () => {
window.removeEventListener("keydown", onKeyDown);
window.removeEventListener("keyup", onKeyUp);
window.removeEventListener("blur", onBlur);
};
}, [
groupCall,
startTalking,
stopTalking,
activeSpeakerUserId,
isAdmin,
talkOverEnabled,
pttButtonHeld,
enablePTTButton,
setMicMuteWrapper,
client,
onClientSync,
]);
useEffect(() => {
client.on(ClientEvent.Sync, onClientSync);

View file

@ -15,6 +15,8 @@ limitations under the License.
*/
import React from "react";
import { Item } from "@react-stately/collections";
import { Modal } from "../Modal";
import styles from "./SettingsModal.module.css";
import { TabContainer, TabItem } from "../tabs/Tabs";
@ -22,7 +24,6 @@ import { ReactComponent as AudioIcon } from "../icons/Audio.svg";
import { ReactComponent as VideoIcon } from "../icons/Video.svg";
import { ReactComponent as DeveloperIcon } from "../icons/Developer.svg";
import { SelectInput } from "../input/SelectInput";
import { Item } from "@react-stately/collections";
import { useMediaHandler } from "./useMediaHandler";
import { useSpatialAudio, useShowInspector } from "./useSetting";
import { FieldRow, InputField } from "../input/Input";
@ -30,7 +31,13 @@ import { Button } from "../button";
import { useDownloadDebugLog } from "./submit-rageshake";
import { Body } from "../typography/Typography";
export const SettingsModal = (props) => {
interface Props {
setShowInspector: boolean;
showInspector: boolean;
[rest: string]: unknown;
}
export const SettingsModal = (props: Props) => {
const {
audioInput,
audioInputs,
@ -42,6 +49,7 @@ export const SettingsModal = (props) => {
audioOutputs,
setAudioOutput,
} = useMediaHandler();
const [spatialAudio, setSpatialAudio] = useSpatialAudio();
const [showInspector, setShowInspector] = useShowInspector();
@ -91,7 +99,9 @@ export const SettingsModal = (props) => {
type="checkbox"
checked={spatialAudio}
description="This will make a speaker's audio seem as if it is coming from where their tile is positioned on screen. (Experimental feature: this may impact the stability of audio.)"
onChange={(e) => setSpatialAudio(e.target.checked)}
onChange={(event: React.ChangeEvent<HTMLInputElement>) =>
setSpatialAudio(event.target.checked)
}
/>
</FieldRow>
</TabItem>
@ -133,7 +143,9 @@ export const SettingsModal = (props) => {
label="Show Call Inspector"
type="checkbox"
checked={showInspector}
onChange={(e) => setShowInspector(e.target.checked)}
onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
setShowInspector(e.target.checked)
}
/>
</FieldRow>
<FieldRow>

View file

@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
/*
Copyright 2017 OpenMarket Ltd
Copyright 2018 New Vector Ltd
@ -37,19 +38,33 @@ limitations under the License.
// actually timestamps. We then purge the remaining logs. We also do this
// purge on startup to prevent logs from accumulating.
// the frequency with which we flush to indexeddb
import { logger } from "matrix-js-sdk/src/logger";
import { randomString } from "matrix-js-sdk/src/randomstring";
// the frequency with which we flush to indexeddb
const FLUSH_RATE_MS = 30 * 1000;
// the length of log data we keep in indexeddb (and include in the reports)
const MAX_LOG_SIZE = 1024 * 1024 * 5; // 5 MB
// A class which monkey-patches the global console and stores log lines.
export class ConsoleLogger {
logs = "";
type LogFunction = (
...args: (Error | DOMException | object | string)[]
) => void;
type LogFunctionName = "log" | "info" | "warn" | "error";
monkeyPatch(consoleObj) {
// A class which monkey-patches the global console and stores log lines.
interface LogEntry {
id: string;
lines: string;
index?: number;
}
export class ConsoleLogger {
private logs = "";
private originalFunctions: { [key in LogFunctionName]?: LogFunction } = {};
public monkeyPatch(consoleObj: Console): void {
// Monkey-patch console logging
const consoleFunctionsToLevels = {
log: "I",
@ -60,6 +75,7 @@ export class ConsoleLogger {
Object.keys(consoleFunctionsToLevels).forEach((fnName) => {
const level = consoleFunctionsToLevels[fnName];
const originalFn = consoleObj[fnName].bind(consoleObj);
this.originalFunctions[fnName] = originalFn;
consoleObj[fnName] = (...args) => {
this.log(level, ...args);
originalFn(...args);
@ -67,7 +83,17 @@ export class ConsoleLogger {
});
}
log(level, ...args) {
public bypassRageshake(
fnName: LogFunctionName,
...args: (Error | DOMException | object | string)[]
): void {
this.originalFunctions[fnName](...args);
}
public log(
level: string,
...args: (Error | DOMException | object | string)[]
): void {
// We don't know what locale the user may be running so use ISO strings
const ts = new Date().toISOString();
@ -78,21 +104,7 @@ export class ConsoleLogger {
} else if (arg instanceof Error) {
return arg.message + (arg.stack ? `\n${arg.stack}` : "");
} else if (typeof arg === "object") {
try {
return JSON.stringify(arg);
} catch (e) {
// In development, it can be useful to log complex cyclic
// objects to the console for inspection. This is fine for
// the console, but default `stringify` can't handle that.
// We workaround this by using a special replacer function
// to only log values of the root object and avoid cycles.
return JSON.stringify(arg, (key, value) => {
if (key && typeof value === "object") {
return "<object>";
}
return value;
});
}
return JSON.stringify(arg, getCircularReplacer());
} else {
return arg;
}
@ -116,7 +128,7 @@ export class ConsoleLogger {
* @param {boolean} keepLogs True to not delete logs after flushing.
* @return {string} \n delimited log lines to flush.
*/
flush(keepLogs) {
public flush(keepLogs?: boolean): string {
// The ConsoleLogger doesn't care how these end up on disk, it just
// flushes them to the caller.
if (keepLogs) {
@ -130,24 +142,23 @@ export class ConsoleLogger {
// A class which stores log lines in an IndexedDB instance.
export class IndexedDBLogStore {
index = 0;
db = null;
flushPromise = null;
flushAgainPromise = null;
private index = 0;
private db: IDBDatabase = null;
private flushPromise: Promise<void> = null;
private flushAgainPromise: Promise<void> = null;
private id: string;
constructor(indexedDB, logger) {
this.indexedDB = indexedDB;
this.logger = logger;
this.id = "instance-" + Math.random() + Date.now();
constructor(private indexedDB: IDBFactory, private logger: ConsoleLogger) {
this.id = "instance-" + randomString(16);
}
/**
* @return {Promise} Resolves when the store is ready.
*/
connect() {
public connect(): Promise<void> {
const req = this.indexedDB.open("logs");
return new Promise((resolve, reject) => {
req.onsuccess = (event) => {
req.onsuccess = (event: Event) => {
// @ts-ignore
this.db = event.target.result;
// Periodically flush logs to local storage / indexeddb
@ -206,7 +217,7 @@ export class IndexedDBLogStore {
*
* @return {Promise} Resolved when the logs have been flushed.
*/
flush() {
public flush(): Promise<void> {
// check if a flush() operation is ongoing
if (this.flushPromise) {
if (this.flushAgainPromise) {
@ -225,7 +236,7 @@ export class IndexedDBLogStore {
}
// there is no flush promise or there was but it has finished, so do
// a brand new one, destroying the chain which may have been built up.
this.flushPromise = new Promise((resolve, reject) => {
this.flushPromise = new Promise<void>((resolve, reject) => {
if (!this.db) {
// not connected yet or user rejected access for us to r/w to the db.
reject(new Error("No connected database"));
@ -243,6 +254,7 @@ export class IndexedDBLogStore {
};
txn.onerror = (event) => {
logger.error("Failed to flush logs : ", event);
// @ts-ignore
reject(new Error("Failed to write logs: " + event.target.errorCode));
};
objStore.add(this.generateLogEntry(lines));
@ -264,12 +276,12 @@ export class IndexedDBLogStore {
* log ID). The objects have said log ID in an "id" field and "lines" which
* is a big string with all the new-line delimited logs.
*/
async consume() {
public async consume(): Promise<LogEntry[]> {
const db = this.db;
// Returns: a string representing the concatenated logs for this ID.
// Stops adding log fragments when the size exceeds maxSize
function fetchLogs(id, maxSize) {
function fetchLogs(id: string, maxSize: number): Promise<string> {
const objectStore = db
.transaction("logs", "readonly")
.objectStore("logs");
@ -280,9 +292,11 @@ export class IndexedDBLogStore {
.openCursor(IDBKeyRange.only(id), "prev");
let lines = "";
query.onerror = (event) => {
// @ts-ignore
reject(new Error("Query failed: " + event.target.errorCode));
};
query.onsuccess = (event) => {
// @ts-ignore
const cursor = event.target.result;
if (!cursor) {
resolve(lines);
@ -299,12 +313,12 @@ export class IndexedDBLogStore {
}
// Returns: A sorted array of log IDs. (newest first)
function fetchLogIds() {
function fetchLogIds(): Promise<string[]> {
// To gather all the log IDs, query for all records in logslastmod.
const o = db
.transaction("logslastmod", "readonly")
.objectStore("logslastmod");
return selectQuery(o, undefined, (cursor) => {
return selectQuery<{ ts: number; id: string }>(o, undefined, (cursor) => {
return {
id: cursor.value.id,
ts: cursor.value.ts,
@ -319,13 +333,14 @@ export class IndexedDBLogStore {
});
}
function deleteLogs(id) {
return new Promise((resolve, reject) => {
function deleteLogs(id: number): Promise<void> {
return new Promise<void>((resolve, reject) => {
const txn = db.transaction(["logs", "logslastmod"], "readwrite");
const o = txn.objectStore("logs");
// only load the key path, not the data which may be huge
const query = o.index("id").openKeyCursor(IDBKeyRange.only(id));
query.onsuccess = (event) => {
// @ts-ignore
const cursor = event.target.result;
if (!cursor) {
return;
@ -340,6 +355,7 @@ export class IndexedDBLogStore {
reject(
new Error(
"Failed to delete logs for " +
// @ts-ignore
`'${id}' : ${event.target.errorCode}`
)
);
@ -352,7 +368,7 @@ export class IndexedDBLogStore {
const allLogIds = await fetchLogIds();
let removeLogIds = [];
const logs = [];
const logs: LogEntry[] = [];
let size = 0;
for (let i = 0; i < allLogIds.length; i++) {
const lines = await fetchLogs(allLogIds[i], MAX_LOG_SIZE - size);
@ -390,7 +406,7 @@ export class IndexedDBLogStore {
return logs;
}
generateLogEntry(lines) {
private generateLogEntry(lines: string): LogEntry {
return {
id: this.id,
lines: lines,
@ -398,7 +414,7 @@ export class IndexedDBLogStore {
};
}
generateLastModifiedTime() {
private generateLastModifiedTime(): { id: string; ts: number } {
return {
id: this.id,
ts: Date.now(),
@ -416,7 +432,11 @@ export class IndexedDBLogStore {
* @return {Promise<T[]>} Resolves to an array of whatever you returned from
* resultMapper.
*/
function selectQuery(store, keyRange, resultMapper) {
function selectQuery<T>(
store: IDBObjectStore,
keyRange: IDBKeyRange,
resultMapper: (cursor: IDBCursorWithValue) => T
): Promise<T[]> {
const query = store.openCursor(keyRange);
return new Promise((resolve, reject) => {
const results = [];
@ -437,6 +457,16 @@ function selectQuery(store, keyRange, resultMapper) {
};
});
}
declare global {
// eslint-disable-next-line no-var, camelcase
var mx_rage_store: IndexedDBLogStore;
// eslint-disable-next-line no-var, camelcase
var mx_rage_logger: ConsoleLogger;
// eslint-disable-next-line no-var, camelcase
var mx_rage_initPromise: Promise<void>;
// eslint-disable-next-line no-var, camelcase
var mx_rage_initStoragePromise: Promise<void>;
}
/**
* Configure rage shaking support for sending bug reports.
@ -445,7 +475,7 @@ function selectQuery(store, keyRange, resultMapper) {
* be set up immediately for the logs.
* @return {Promise} Resolves when set up.
*/
export function init(setUpPersistence = true) {
export function init(setUpPersistence = true): Promise<void> {
if (global.mx_rage_initPromise) {
return global.mx_rage_initPromise;
}
@ -465,7 +495,7 @@ export function init(setUpPersistence = true) {
* then this no-ops.
* @return {Promise} Resolves when complete.
*/
export function tryInitStorage() {
export function tryInitStorage(): Promise<void> {
if (global.mx_rage_initStoragePromise) {
return global.mx_rage_initStoragePromise;
}
@ -491,7 +521,7 @@ export function tryInitStorage() {
return global.mx_rage_initStoragePromise;
}
export function flush() {
export function flush(): Promise<void> {
if (!global.mx_rage_store) {
return;
}
@ -502,7 +532,7 @@ export function flush() {
* Clean up old logs.
* @return {Promise} Resolves if cleaned logs.
*/
export async function cleanup() {
export async function cleanup(): Promise<void> {
if (!global.mx_rage_store) {
return;
}
@ -512,9 +542,9 @@ export async function cleanup() {
/**
* Get a recent snapshot of the logs, ready for attaching to a bug report
*
* @return {Array<{lines: string, id, string}>} list of log data
* @return {LogEntry[]} list of log data
*/
export async function getLogsForReport() {
export async function getLogsForReport(): Promise<LogEntry[]> {
if (!global.mx_rage_logger) {
throw new Error("No console logger, did you forget to call init()?");
}
@ -523,7 +553,7 @@ export async function getLogsForReport() {
if (global.mx_rage_store) {
// flush most recent logs
await global.mx_rage_store.flush();
return await global.mx_rage_store.consume();
return global.mx_rage_store.consume();
} else {
return [
{
@ -533,3 +563,24 @@ export async function getLogsForReport() {
];
}
}
type StringifyReplacer = (
this: unknown,
key: string,
value: unknown
) => unknown;
// From https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Cyclic_object_value#circular_references
// Injects `<$ cycle-trimmed $>` wherever it cuts a cyclical object relationship
const getCircularReplacer = (): StringifyReplacer => {
const seen = new WeakSet();
return (key: string, value: unknown): unknown => {
if (typeof value === "object" && value !== null) {
if (seen.has(value)) {
return "<$ cycle-trimmed $>";
}
seen.add(value);
}
return value;
};
};

View file

@ -15,14 +15,31 @@ limitations under the License.
*/
import { useCallback, useContext, useEffect, useState } from "react";
import { getLogsForReport } from "./rageshake";
import pako from "pako";
import { MatrixEvent } from "matrix-js-sdk";
import { OverlayTriggerState } from "@react-stately/overlays";
import { MatrixClient, ClientEvent } from "matrix-js-sdk/src/client";
import { getLogsForReport } from "./rageshake";
import { useClient } from "../ClientContext";
import { InspectorContext } from "../room/GroupCallInspector";
import { useModalTriggerState } from "../Modal";
export function useSubmitRageshake() {
const { client } = useClient();
interface RageShakeSubmitOptions {
description: string;
roomId: string;
label: string;
sendLogs: boolean;
rageshakeRequestId: string;
}
export function useSubmitRageshake(): {
submitRageshake: (opts: RageShakeSubmitOptions) => Promise<void>;
sending: boolean;
sent: boolean;
error: Error;
} {
const client: MatrixClient = useClient().client;
const [{ json }] = useContext(InspectorContext);
const [{ sending, sent, error }, setState] = useState({
@ -57,9 +74,12 @@ export function useSubmitRageshake() {
opts.description || "User did not supply any additional text."
);
body.append("app", "matrix-video-chat");
body.append("version", import.meta.env.VITE_APP_VERSION || "dev");
body.append(
"version",
(import.meta.env.VITE_APP_VERSION as string) || "dev"
);
body.append("user_agent", userAgent);
body.append("installed_pwa", false);
body.append("installed_pwa", "false");
body.append("touch_input", touchInput);
if (client) {
@ -181,7 +201,11 @@ export function useSubmitRageshake() {
if (navigator.storage && navigator.storage.estimate) {
try {
const estimate = await navigator.storage.estimate();
const estimate: {
quota?: number;
usage?: number;
usageDetails?: { [x: string]: unknown };
} = await navigator.storage.estimate();
body.append("storageManager_quota", String(estimate.quota));
body.append("storageManager_usage", String(estimate.usage));
if (estimate.usageDetails) {
@ -201,7 +225,6 @@ export function useSubmitRageshake() {
for (const entry of logs) {
// encode as UTF-8
let buf = new TextEncoder().encode(entry.lines);
// compress
buf = pako.gzip(buf);
@ -225,7 +248,7 @@ export function useSubmitRageshake() {
}
await fetch(
import.meta.env.VITE_RAGESHAKE_SUBMIT_URL ||
(import.meta.env.VITE_RAGESHAKE_SUBMIT_URL as string) ||
"https://element.io/bugreports/submit",
{
method: "POST",
@ -250,7 +273,7 @@ export function useSubmitRageshake() {
};
}
export function useDownloadDebugLog() {
export function useDownloadDebugLog(): () => void {
const [{ json }] = useContext(InspectorContext);
const downloadDebugLog = useCallback(() => {
@ -271,7 +294,10 @@ export function useDownloadDebugLog() {
return downloadDebugLog;
}
export function useRageshakeRequest() {
export function useRageshakeRequest(): (
roomId: string,
rageshakeRequestId: string
) => void {
const { client } = useClient();
const sendRageshakeRequest = useCallback(
@ -285,14 +311,27 @@ export function useRageshakeRequest() {
return sendRageshakeRequest;
}
interface ModalProps {
isOpen: boolean;
onClose: () => void;
}
interface ModalPropsWithId extends ModalProps {
rageshakeRequestId: string;
}
export function useRageshakeRequestModal(roomId) {
const { modalState, modalProps } = useModalTriggerState();
const { client } = useClient();
const [rageshakeRequestId, setRageshakeRequestId] = useState();
export function useRageshakeRequestModal(roomId: string): {
modalState: OverlayTriggerState;
modalProps: ModalPropsWithId;
} {
const { modalState, modalProps } = useModalTriggerState() as {
modalState: OverlayTriggerState;
modalProps: ModalProps;
};
const client: MatrixClient = useClient().client;
const [rageshakeRequestId, setRageshakeRequestId] = useState<string>();
useEffect(() => {
const onEvent = (event) => {
const onEvent = (event: MatrixEvent) => {
const type = event.getType();
if (
@ -305,10 +344,10 @@ export function useRageshakeRequestModal(roomId) {
}
};
client.on("event", onEvent);
client.on(ClientEvent.Event, onEvent);
return () => {
client.removeListener("event", onEvent);
client.removeListener(ClientEvent.Event, onEvent);
};
}, [modalState.open, roomId, client, modalState]);

View file

@ -1,3 +1,4 @@
/* eslint-disable @typescript-eslint/ban-ts-comment */
/*
Copyright 2022 Matrix.org Foundation C.I.C.
@ -14,6 +15,8 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import { MatrixClient } from "matrix-js-sdk";
import { MediaHandlerEvent } from "matrix-js-sdk/src/webrtc/mediaHandler";
import React, {
useState,
useEffect,
@ -23,9 +26,27 @@ import React, {
createContext,
} from "react";
const MediaHandlerContext = createContext();
export interface MediaHandlerContextInterface {
audioInput: string;
audioInputs: MediaDeviceInfo[];
setAudioInput: (deviceId: string) => void;
videoInput: string;
videoInputs: MediaDeviceInfo[];
setVideoInput: (deviceId: string) => void;
audioOutput: string;
audioOutputs: MediaDeviceInfo[];
setAudioOutput: (deviceId: string) => void;
}
function getMediaPreferences() {
const MediaHandlerContext =
createContext<MediaHandlerContextInterface>(undefined);
interface MediaPreferences {
audioInput?: string;
videoInput?: string;
audioOutput?: string;
}
function getMediaPreferences(): MediaPreferences {
const mediaPreferences = localStorage.getItem("matrix-media-preferences");
if (mediaPreferences) {
@ -39,8 +60,8 @@ function getMediaPreferences() {
}
}
function updateMediaPreferences(newPreferences) {
const oldPreferences = getMediaPreferences(newPreferences);
function updateMediaPreferences(newPreferences: MediaPreferences): void {
const oldPreferences = getMediaPreferences();
localStorage.setItem(
"matrix-media-preferences",
@ -50,8 +71,11 @@ function updateMediaPreferences(newPreferences) {
})
);
}
export function MediaHandlerProvider({ client, children }) {
interface Props {
client: MatrixClient;
children: JSX.Element[];
}
export function MediaHandlerProvider({ client, children }: Props): JSX.Element {
const [
{
audioInput,
@ -72,7 +96,9 @@ export function MediaHandlerProvider({ client, children }) {
);
return {
// @ts-ignore, ignore that audioInput is a private members of mediaHandler
audioInput: mediaHandler.audioInput,
// @ts-ignore, ignore that videoInput is a private members of mediaHandler
videoInput: mediaHandler.videoInput,
audioOutput: undefined,
audioInputs: [],
@ -84,7 +110,7 @@ export function MediaHandlerProvider({ client, children }) {
useEffect(() => {
const mediaHandler = client.getMediaHandler();
function updateDevices() {
function updateDevices(): void {
navigator.mediaDevices.enumerateDevices().then((devices) => {
const mediaPreferences = getMediaPreferences();
@ -92,9 +118,10 @@ export function MediaHandlerProvider({ client, children }) {
(device) => device.kind === "audioinput"
);
const audioConnected = audioInputs.some(
// @ts-ignore
(device) => device.deviceId === mediaHandler.audioInput
);
// @ts-ignore
let audioInput = mediaHandler.audioInput;
if (!audioConnected && audioInputs.length > 0) {
@ -105,9 +132,11 @@ export function MediaHandlerProvider({ client, children }) {
(device) => device.kind === "videoinput"
);
const videoConnected = videoInputs.some(
// @ts-ignore
(device) => device.deviceId === mediaHandler.videoInput
);
// @ts-ignore
let videoInput = mediaHandler.videoInput;
if (!videoConnected && videoInputs.length > 0) {
@ -129,7 +158,9 @@ export function MediaHandlerProvider({ client, children }) {
}
if (
// @ts-ignore
mediaHandler.videoInput !== videoInput ||
// @ts-ignore
mediaHandler.audioInput !== audioInput
) {
mediaHandler.setMediaInputs(audioInput, videoInput);
@ -149,18 +180,21 @@ export function MediaHandlerProvider({ client, children }) {
}
updateDevices();
mediaHandler.on("local_streams_changed", updateDevices);
mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, updateDevices);
navigator.mediaDevices.addEventListener("devicechange", updateDevices);
return () => {
mediaHandler.removeListener("local_streams_changed", updateDevices);
mediaHandler.removeListener(
MediaHandlerEvent.LocalStreamsChanged,
updateDevices
);
navigator.mediaDevices.removeEventListener("devicechange", updateDevices);
mediaHandler.stopAllStreams();
};
}, [client]);
const setAudioInput = useCallback(
(deviceId) => {
const setAudioInput: (deviceId: string) => void = useCallback(
(deviceId: string) => {
updateMediaPreferences({ audioInput: deviceId });
setState((prevState) => ({ ...prevState, audioInput: deviceId }));
client.getMediaHandler().setAudioInput(deviceId);
@ -168,7 +202,7 @@ export function MediaHandlerProvider({ client, children }) {
[client]
);
const setVideoInput = useCallback(
const setVideoInput: (deviceId: string) => void = useCallback(
(deviceId) => {
updateMediaPreferences({ videoInput: deviceId });
setState((prevState) => ({ ...prevState, videoInput: deviceId }));
@ -177,35 +211,36 @@ export function MediaHandlerProvider({ client, children }) {
[client]
);
const setAudioOutput = useCallback((deviceId) => {
const setAudioOutput: (deviceId: string) => void = useCallback((deviceId) => {
updateMediaPreferences({ audioOutput: deviceId });
setState((prevState) => ({ ...prevState, audioOutput: deviceId }));
}, []);
const context = useMemo(
() => ({
audioInput,
audioInputs,
setAudioInput,
videoInput,
videoInputs,
setVideoInput,
audioOutput,
audioOutputs,
setAudioOutput,
}),
[
audioInput,
audioInputs,
setAudioInput,
videoInput,
videoInputs,
setVideoInput,
audioOutput,
audioOutputs,
setAudioOutput,
]
);
const context: MediaHandlerContextInterface =
useMemo<MediaHandlerContextInterface>(
() => ({
audioInput,
audioInputs,
setAudioInput,
videoInput,
videoInputs,
setVideoInput,
audioOutput,
audioOutputs,
setAudioOutput,
}),
[
audioInput,
audioInputs,
setAudioInput,
videoInput,
videoInputs,
setVideoInput,
audioOutput,
audioOutputs,
setAudioOutput,
]
);
return (
<MediaHandlerContext.Provider value={context}>

View file

@ -42,7 +42,7 @@ export const PTTClips: React.FC<Props> = ({
return (
<>
<audio
preload="true"
preload="auto"
className={styles.pttClip}
ref={startTalkingLocalRef}
>
@ -50,18 +50,18 @@ export const PTTClips: React.FC<Props> = ({
<source type="audio/mpeg" src={startTalkLocalMp3Url} />
</audio>
<audio
preload="true"
preload="auto"
className={styles.pttClip}
ref={startTalkingRemoteRef}
>
<source type="audio/ogg" src={startTalkRemoteOggUrl} />
<source type="audio/mpeg" src={startTalkRemoteMp3Url} />
</audio>
<audio preload="true" className={styles.pttClip} ref={endTalkingRef}>
<audio preload="auto" className={styles.pttClip} ref={endTalkingRef}>
<source type="audio/ogg" src={endTalkOggUrl} />
<source type="audio/mpeg" src={endTalkMp3Url} />
</audio>
<audio preload="true" className={styles.pttClip} ref={blockedRef}>
<audio preload="auto" className={styles.pttClip} ref={blockedRef}>
<source type="audio/ogg" src={blockedOggUrl} />
<source type="audio/mpeg" src={blockedMp3Url} />
</audio>

View file

@ -58,8 +58,12 @@ export const usePTTSounds = (): PTTSounds => {
break;
}
if (ref.current) {
ref.current.currentTime = 0;
await ref.current.play();
try {
ref.current.currentTime = 0;
await ref.current.play();
} catch (e) {
console.log("Couldn't play sound effect", e);
}
} else {
console.log("No media element found");
}

49
src/useDelayedState.ts Normal file
View file

@ -0,0 +1,49 @@
/*
Copyright 2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { useState, useRef, useCallback } from "react";
// Like useState, except state updates can be enqueued with a configurable delay
export const useDelayedState = <T>(
initial?: T
): [T, (value: T, delay: number) => void, (value: T) => void] => {
const [state, setState] = useState<T>(initial);
const timers = useRef<Set<ReturnType<typeof setTimeout>>>();
if (!timers.current) timers.current = new Set();
const setStateDelayed = useCallback(
(value: T, delay: number) => {
const timer = setTimeout(() => {
setState(value);
timers.current.delete(timer);
}, delay);
timers.current.add(timer);
},
[setState, timers]
);
const setStateImmediate = useCallback(
(value: T) => {
// Clear all updates currently in the queue
for (const timer of timers.current) clearTimeout(timer);
timers.current.clear();
setState(value);
},
[setState, timers]
);
return [state, setStateDelayed, setStateImmediate];
};

34
src/useEvents.ts Normal file
View file

@ -0,0 +1,34 @@
/*
Copyright 2022 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { useEffect } from "react";
// Shortcut for registering a listener on an EventTarget
export const useEventTarget = <T extends Event>(
target: EventTarget,
eventType: string,
listener: (event: T) => void,
options?: AddEventListenerOptions
) => {
useEffect(() => {
if (target) {
target.addEventListener(eventType, listener, options);
return () => target.removeEventListener(eventType, listener, options);
}
}, [target, eventType, listener, options]);
};
// TODO: Have a similar hook for EventEmitters

View file

@ -29,6 +29,7 @@ export function VideoTileContainer({
showName,
audioOutputDevice,
audioContext,
audioDestination,
disableSpeakingIndicator,
...rest
}) {
@ -47,6 +48,7 @@ export function VideoTileContainer({
stream,
audioOutputDevice,
audioContext,
audioDestination,
isLocal
);

View file

@ -1,144 +0,0 @@
/*
Copyright 2022 Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { useRef, useEffect } from "react";
import { useSpatialAudio } from "../settings/useSetting";
export function useMediaStream(stream, audioOutputDevice, mute = false) {
const mediaRef = useRef();
useEffect(() => {
console.log(
`useMediaStream update stream mediaRef.current ${!!mediaRef.current} stream ${
stream && stream.id
}`
);
if (mediaRef.current) {
const mediaEl = mediaRef.current;
if (stream) {
mediaEl.muted = mute;
mediaEl.srcObject = stream;
mediaEl.play();
// Unmuting the tab in Safari causes all video elements to be individually
// unmuted, so we need to reset the mute state here to prevent audio loops
const onVolumeChange = () => {
mediaEl.muted = mute;
};
mediaEl.addEventListener("volumechange", onVolumeChange);
return () =>
mediaEl.removeEventListener("volumechange", onVolumeChange);
} else {
mediaRef.current.srcObject = null;
}
}
}, [stream, mute]);
useEffect(() => {
if (
mediaRef.current &&
audioOutputDevice &&
mediaRef.current !== undefined
) {
console.log(`useMediaStream setSinkId ${audioOutputDevice}`);
// Chrome for Android doesn't support this
mediaRef.current.setSinkId?.(audioOutputDevice);
}
}, [audioOutputDevice]);
useEffect(() => {
const mediaEl = mediaRef.current;
return () => {
if (mediaEl) {
// Ensure we set srcObject to null before unmounting to prevent memory leak
// https://webrtchacks.com/srcobject-intervention/
mediaEl.srcObject = null;
}
};
}, []);
return mediaRef;
}
export const useSpatialMediaStream = (
stream,
audioOutputDevice,
audioContext,
mute = false
) => {
const tileRef = useRef();
const [spatialAudio] = useSpatialAudio();
// If spatial audio is enabled, we handle audio separately from the video element
const mediaRef = useMediaStream(
stream,
audioOutputDevice,
spatialAudio || mute
);
const pannerNodeRef = useRef();
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
refDistance: 3,
});
}
const sourceRef = useRef();
useEffect(() => {
if (spatialAudio && tileRef.current && !mute) {
if (!sourceRef.current) {
sourceRef.current = audioContext.createMediaStreamSource(stream);
}
const tile = tileRef.current;
const source = sourceRef.current;
const pannerNode = pannerNodeRef.current;
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
// Position the source relative to its placement in the window
pannerNodeRef.current.positionX.value =
(bounds.x + bounds.width / 2) / windowSize - 0.5;
pannerNodeRef.current.positionY.value =
(bounds.y + bounds.height / 2) / windowSize - 0.5;
// Put the source in front of the listener
pannerNodeRef.current.positionZ.value = -2;
};
updatePosition();
source.connect(pannerNode);
pannerNode.connect(audioContext.destination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
// moves, because useMeasure, IntersectionObserver, etc. all have no
// ability to track changes in the CSS transform property
tile.addEventListener("transitionrun", updatePosition);
return () => {
tile.removeEventListener("transitionrun", updatePosition);
source.disconnect();
pannerNode.disconnect();
};
}
}, [stream, spatialAudio, audioContext, mute]);
return [tileRef, mediaRef];
};

View file

@ -0,0 +1,248 @@
/*
Copyright 2022 Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { useRef, useEffect, RefObject } from "react";
import { parse as parseSdp, write as writeSdp } from "sdp-transform";
import {
acquireContext,
releaseContext,
} from "matrix-js-sdk/src/webrtc/audioContext";
import { useSpatialAudio } from "../settings/useSetting";
declare global {
interface Window {
// For detecting whether this browser is Chrome or not
chrome?: unknown;
}
}
export const useMediaStream = (
stream: MediaStream,
audioOutputDevice: string,
mute = false
): RefObject<MediaElement> => {
const mediaRef = useRef<MediaElement>();
useEffect(() => {
console.log(
`useMediaStream update stream mediaRef.current ${!!mediaRef.current} stream ${
stream && stream.id
}`
);
if (mediaRef.current) {
const mediaEl = mediaRef.current;
if (stream) {
mediaEl.muted = mute;
mediaEl.srcObject = stream;
mediaEl.play();
// Unmuting the tab in Safari causes all video elements to be individually
// unmuted, so we need to reset the mute state here to prevent audio loops
const onVolumeChange = () => {
mediaEl.muted = mute;
};
mediaEl.addEventListener("volumechange", onVolumeChange);
return () =>
mediaEl.removeEventListener("volumechange", onVolumeChange);
} else {
mediaRef.current.srcObject = null;
}
}
}, [stream, mute]);
useEffect(() => {
if (
mediaRef.current &&
audioOutputDevice &&
mediaRef.current !== undefined
) {
if (mediaRef.current.setSinkId) {
console.log(
`useMediaStream setting output setSinkId ${audioOutputDevice}`
);
// Chrome for Android doesn't support this
mediaRef.current.setSinkId(audioOutputDevice);
} else {
console.log("Can't set output - no setsinkid");
}
}
}, [audioOutputDevice]);
useEffect(() => {
const mediaEl = mediaRef.current;
return () => {
if (mediaEl) {
// Ensure we set srcObject to null before unmounting to prevent memory leak
// https://webrtchacks.com/srcobject-intervention/
mediaEl.srcObject = null;
}
};
}, []);
return mediaRef;
};
// Loops the given audio stream back through a local peer connection, to make
// AEC work with Web Audio streams on Chrome. The resulting stream should be
// played through an audio element.
// This hack can be removed once the following bug is resolved:
// https://bugs.chromium.org/p/chromium/issues/detail?id=687574
const createLoopback = async (stream: MediaStream): Promise<MediaStream> => {
// Prepare our local peer connections
const conn = new RTCPeerConnection();
const loopbackConn = new RTCPeerConnection();
const loopbackStream = new MediaStream();
conn.addEventListener("icecandidate", ({ candidate }) => {
if (candidate) loopbackConn.addIceCandidate(new RTCIceCandidate(candidate));
});
loopbackConn.addEventListener("icecandidate", ({ candidate }) => {
if (candidate) conn.addIceCandidate(new RTCIceCandidate(candidate));
});
loopbackConn.addEventListener("track", ({ track }) =>
loopbackStream.addTrack(track)
);
// Hook the connections together
stream.getTracks().forEach((track) => conn.addTrack(track));
const offer = await conn.createOffer({
offerToReceiveAudio: false,
offerToReceiveVideo: false,
});
await conn.setLocalDescription(offer);
await loopbackConn.setRemoteDescription(offer);
const answer = await loopbackConn.createAnswer();
// Rewrite SDP to be stereo and (variable) max bitrate
const parsedSdp = parseSdp(answer.sdp);
parsedSdp.media.forEach((m) =>
m.fmtp.forEach(
(f) => (f.config += `;stereo=1;cbr=0;maxaveragebitrate=510000;`)
)
);
answer.sdp = writeSdp(parsedSdp);
await loopbackConn.setLocalDescription(answer);
await conn.setRemoteDescription(answer);
return loopbackStream;
};
export const useAudioContext = (): [
AudioContext,
AudioNode,
RefObject<HTMLAudioElement>
] => {
const context = useRef<AudioContext>();
const destination = useRef<AudioNode>();
const audioRef = useRef<HTMLAudioElement>();
useEffect(() => {
if (audioRef.current && !context.current) {
context.current = acquireContext();
if (window.chrome) {
// We're in Chrome, which needs a loopback hack applied to enable AEC
const streamDest = context.current.createMediaStreamDestination();
destination.current = streamDest;
const audioEl = audioRef.current;
(async () => {
audioEl.srcObject = await createLoopback(streamDest.stream);
await audioEl.play();
})();
return () => {
audioEl.srcObject = null;
releaseContext();
};
} else {
destination.current = context.current.destination;
return releaseContext;
}
}
}, []);
return [context.current, destination.current, audioRef];
};
export const useSpatialMediaStream = (
stream: MediaStream,
audioOutputDevice: string,
audioContext: AudioContext,
audioDestination: AudioNode,
mute = false
): [RefObject<Element>, RefObject<MediaElement>] => {
const tileRef = useRef<Element>();
const [spatialAudio] = useSpatialAudio();
// If spatial audio is enabled, we handle audio separately from the video element
const mediaRef = useMediaStream(
stream,
audioOutputDevice,
spatialAudio || mute
);
const pannerNodeRef = useRef<PannerNode>();
const sourceRef = useRef<MediaStreamAudioSourceNode>();
useEffect(() => {
if (spatialAudio && tileRef.current && !mute) {
if (!pannerNodeRef.current) {
pannerNodeRef.current = new PannerNode(audioContext, {
panningModel: "HRTF",
refDistance: 3,
});
}
if (!sourceRef.current) {
sourceRef.current = audioContext.createMediaStreamSource(stream);
}
const tile = tileRef.current;
const source = sourceRef.current;
const pannerNode = pannerNodeRef.current;
const updatePosition = () => {
const bounds = tile.getBoundingClientRect();
const windowSize = Math.max(window.innerWidth, window.innerHeight);
// Position the source relative to its placement in the window
pannerNodeRef.current.positionX.value =
(bounds.x + bounds.width / 2) / windowSize - 0.5;
pannerNodeRef.current.positionY.value =
(bounds.y + bounds.height / 2) / windowSize - 0.5;
// Put the source in front of the listener
pannerNodeRef.current.positionZ.value = -2;
};
updatePosition();
source.connect(pannerNode).connect(audioDestination);
// HACK: We abuse the CSS transitionrun event to detect when the tile
// moves, because useMeasure, IntersectionObserver, etc. all have no
// ability to track changes in the CSS transform property
tile.addEventListener("transitionrun", updatePosition);
return () => {
tile.removeEventListener("transitionrun", updatePosition);
source.disconnect();
pannerNode.disconnect();
};
}
}, [stream, spatialAudio, audioContext, audioDestination, mute]);
return [tileRef, mediaRef];
};

View file

@ -3024,6 +3024,11 @@
resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.2.tgz#1a62f89525723dde24ba1b01b092bf5df8ad4d39"
integrity sha512-hppQEBDmlwhFAXKJX2KnWLYu5yMfi91yazPb2l+lbJiwW+wdo1gNeRA+3RgNSO39WYX2euey41KEwnqesU2Jew==
"@types/sdp-transform@^2.4.5":
version "2.4.5"
resolved "https://registry.yarnpkg.com/@types/sdp-transform/-/sdp-transform-2.4.5.tgz#3167961e0a1a5265545e278627aa37c606003f53"
integrity sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg==
"@types/source-list-map@*":
version "0.1.2"
resolved "https://registry.yarnpkg.com/@types/source-list-map/-/source-list-map-0.1.2.tgz#0078836063ffaf17412349bba364087e0ac02ec9"
@ -8597,11 +8602,12 @@ matrix-events-sdk@^0.0.1-beta.7:
resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1-beta.7.tgz#5ffe45eba1f67cc8d7c2377736c728b322524934"
integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA==
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#404f8e130e44a78b0159c55902df1b129b3816d1":
version "17.2.0"
resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/404f8e130e44a78b0159c55902df1b129b3816d1"
"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f9672cf3076bf5e1868c92ee94c780040b0c6fb1":
version "18.1.0"
resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f9672cf3076bf5e1868c92ee94c780040b0c6fb1"
dependencies:
"@babel/runtime" "^7.12.5"
"@types/sdp-transform" "^2.4.5"
another-json "^0.2.0"
browser-request "^0.3.3"
bs58 "^4.0.1"
@ -8611,6 +8617,7 @@ matrix-events-sdk@^0.0.1-beta.7:
p-retry "^4.5.0"
qs "^6.9.6"
request "^2.88.2"
sdp-transform "^2.14.1"
unhomoglyph "^1.0.6"
md5.js@^1.3.4:
@ -11046,6 +11053,11 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1:
ajv "^6.12.5"
ajv-keywords "^3.5.2"
sdp-transform@^2.14.1:
version "2.14.1"
resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827"
integrity sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw==
"semver@2 || 3 || 4 || 5", semver@^5.4.1, semver@^5.6.0:
version "5.7.1"
resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"