From 313ebe258e20686319fe98234aea5f4ce5390b02 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 22 Mar 2023 14:23:26 -0400 Subject: [PATCH 1/4] Add end-to-end audio observability This reports via OpenTelemetry when particular participants are speaking, as an easy way to observe the delivery of audio in calls. --- src/otel/OTelGroupCallMembership.ts | 45 ++++++++++++++++++++++---- src/room/InCallView.tsx | 2 +- src/video-grid/AudioSink.tsx | 8 ++++- src/video-grid/useCallFeed.ts | 50 +++++++++++++++++------------ 4 files changed, 76 insertions(+), 29 deletions(-) diff --git a/src/otel/OTelGroupCallMembership.ts b/src/otel/OTelGroupCallMembership.ts index 764249f..3a00d43 100644 --- a/src/otel/OTelGroupCallMembership.ts +++ b/src/otel/OTelGroupCallMembership.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import opentelemetry, { Span, Attributes } from "@opentelemetry/api"; +import opentelemetry, { Span, Attributes, Context } from "@opentelemetry/api"; import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions"; import { GroupCall, @@ -73,8 +73,10 @@ function flattenVoipEventRecursive( */ export class OTelGroupCallMembership { private callMembershipSpan?: Span; + private callMembershipContext?: Context; private myUserId: string; private myMember: RoomMember; + private readonly speakingSpans = new Map>(); constructor(private groupCall: GroupCall, client: MatrixClient) { this.myUserId = client.getUserId(); @@ -101,7 +103,7 @@ export class OTelGroupCallMembership { this.myMember.name ); - opentelemetry.trace.setSpan( + this.callMembershipContext = opentelemetry.trace.setSpan( opentelemetry.context.active(), this.callMembershipSpan ); @@ -110,10 +112,11 @@ export class OTelGroupCallMembership { } public onLeaveCall() { - this.callMembershipSpan?.addEvent("matrix.leaveCall"); - - // and end the main span to indicate we've left - if (this.callMembershipSpan) this.callMembershipSpan.end(); + this.callMembershipSpan!.addEvent("matrix.leaveCall"); + // and end the span to indicate we've left + this.callMembershipSpan!.end(); + this.callMembershipSpan = undefined; + this.callMembershipContext = undefined; } public onUpdateRoomState(event: MatrixEvent) { @@ -177,4 +180,34 @@ export class OTelGroupCallMembership { "matrix.screensharing.enabled": newValue, }); } + + public onSpeaking(member: RoomMember, deviceId: string, speaking: boolean) { + if (speaking) { + // Ensure that there's an audio activity span for this speaker + let deviceMap = this.speakingSpans.get(member); + if (deviceMap === undefined) { + deviceMap = new Map(); + this.speakingSpans.set(member, deviceMap); + } + + if (!deviceMap.has(deviceId)) { + const span = ElementCallOpenTelemetry.instance.tracer.startSpan( + "matrix.audioActivity", + undefined, + this.callMembershipContext + ); + span.setAttribute("matrix.userId", member.userId); + span.setAttribute("matrix.displayName", member.rawDisplayName); + + deviceMap.set(deviceId, span); + } + } else { + // End the audio activity span for this speaker, if any + const deviceMap = this.speakingSpans.get(member); + deviceMap?.get(deviceId)?.end(); + deviceMap?.delete(deviceId); + + if (deviceMap?.size === 0) this.speakingSpans.delete(member); + } + } } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 2e0e932..cf1e4dc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -360,11 +360,11 @@ export function InCallView({ const audioElements: JSX.Element[] = []; if (!spatialAudio || maximisedParticipant) { for (const item of items) { - if (item.isLocal) continue; // We don't want to render own audio audioElements.push( ); diff --git a/src/video-grid/AudioSink.tsx b/src/video-grid/AudioSink.tsx index 24019e4..3787cdc 100644 --- a/src/video-grid/AudioSink.tsx +++ b/src/video-grid/AudioSink.tsx @@ -16,6 +16,7 @@ limitations under the License. import React from "react"; +import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; import { TileDescriptor } from "./TileDescriptor"; import { useCallFeed } from "./useCallFeed"; import { useMediaStream } from "./useMediaStream"; @@ -23,6 +24,7 @@ import { useMediaStream } from "./useMediaStream"; interface Props { tileDescriptor: TileDescriptor; audioOutput: string; + otelGroupCallMembership: OTelGroupCallMembership; } // Renders and