Add end-to-end audio observability

This reports via OpenTelemetry when particular participants are speaking, as an easy way to observe the delivery of audio in calls.

src/otel/OTelGroupCallMembership.ts

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import opentelemetry, { Span, Attributes } from "@opentelemetry/api";
+import opentelemetry, { Span, Attributes, Context } from "@opentelemetry/api";
 import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
   GroupCall,
@@ -73,8 +73,10 @@ function flattenVoipEventRecursive(
  */
 export class OTelGroupCallMembership {
   private callMembershipSpan?: Span;
+  private callMembershipContext?: Context;
   private myUserId: string;
   private myMember: RoomMember;
+  private readonly speakingSpans = new Map<RoomMember, Map<string, Span>>();
 
   constructor(private groupCall: GroupCall, client: MatrixClient) {
     this.myUserId = client.getUserId();
@@ -101,7 +103,7 @@ export class OTelGroupCallMembership {
       this.myMember.name
     );
 
-    opentelemetry.trace.setSpan(
+    this.callMembershipContext = opentelemetry.trace.setSpan(
       opentelemetry.context.active(),
       this.callMembershipSpan
     );
@@ -110,10 +112,11 @@ export class OTelGroupCallMembership {
   }
 
   public onLeaveCall() {
-    this.callMembershipSpan?.addEvent("matrix.leaveCall");
-
-    // and end the main span to indicate we've left
-    if (this.callMembershipSpan) this.callMembershipSpan.end();
+    this.callMembershipSpan!.addEvent("matrix.leaveCall");
+    // and end the span to indicate we've left
+    this.callMembershipSpan!.end();
+    this.callMembershipSpan = undefined;
+    this.callMembershipContext = undefined;
   }
 
   public onUpdateRoomState(event: MatrixEvent) {
@@ -177,4 +180,34 @@ export class OTelGroupCallMembership {
       "matrix.screensharing.enabled": newValue,
     });
   }
+
+  public onSpeaking(member: RoomMember, deviceId: string, speaking: boolean) {
+    if (speaking) {
+      // Ensure that there's an audio activity span for this speaker
+      let deviceMap = this.speakingSpans.get(member);
+      if (deviceMap === undefined) {
+        deviceMap = new Map();
+        this.speakingSpans.set(member, deviceMap);
+      }
+
+      if (!deviceMap.has(deviceId)) {
+        const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
+          "matrix.audioActivity",
+          undefined,
+          this.callMembershipContext
+        );
+        span.setAttribute("matrix.userId", member.userId);
+        span.setAttribute("matrix.displayName", member.rawDisplayName);
+
+        deviceMap.set(deviceId, span);
+      }
+    } else {
+      // End the audio activity span for this speaker, if any
+      const deviceMap = this.speakingSpans.get(member);
+      deviceMap?.get(deviceId)?.end();
+      deviceMap?.delete(deviceId);
+
+      if (deviceMap?.size === 0) this.speakingSpans.delete(member);
+    }
+  }
 }

src/room/InCallView.tsx

@@ -360,11 +360,11 @@ export function InCallView({
   const audioElements: JSX.Element[] = [];
   if (!spatialAudio || maximisedParticipant) {
     for (const item of items) {
-      if (item.isLocal) continue; // We don't want to render own audio
       audioElements.push(
         <AudioSink
           tileDescriptor={item}
           audioOutput={audioOutput}
+          otelGroupCallMembership={otelGroupCallMembership}
           key={item.id}
         />
       );

src/video-grid/AudioSink.tsx

@@ -16,6 +16,7 @@ limitations under the License.
 
 import React from "react";
 
+import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { TileDescriptor } from "./TileDescriptor";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStream } from "./useMediaStream";
@@ -23,6 +24,7 @@ import { useMediaStream } from "./useMediaStream";
 interface Props {
   tileDescriptor: TileDescriptor;
   audioOutput: string;
+  otelGroupCallMembership: OTelGroupCallMembership;
 }
 
 // Renders and <audio> element on the page playing the given stream
@@ -30,8 +32,12 @@ interface Props {
 export const AudioSink: React.FC<Props> = ({
   tileDescriptor,
   audioOutput,
+  otelGroupCallMembership,
 }: Props) => {
-  const { localVolume, stream } = useCallFeed(tileDescriptor.callFeed);
+  const { localVolume, stream } = useCallFeed(
+    tileDescriptor.callFeed,
+    otelGroupCallMembership
+  );
 
   const audioElementRef = useMediaStream(
     stream,

src/video-grid/useCallFeed.ts

@@ -18,6 +18,8 @@ import { useState, useEffect } from "react";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 
+import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
+
 interface CallFeedState {
   callFeed: CallFeed | undefined;
   isLocal: boolean;
@@ -46,40 +48,46 @@ function getCallFeedState(callFeed: CallFeed | undefined): CallFeedState {
   };
 }
 
-export function useCallFeed(callFeed: CallFeed | undefined): CallFeedState {
+export function useCallFeed(
+  callFeed: CallFeed | undefined,
+  otelGroupCallMembership?: OTelGroupCallMembership
+): CallFeedState {
   const [state, setState] = useState<CallFeedState>(() =>
     getCallFeedState(callFeed)
   );
 
   useEffect(() => {
-    function onSpeaking(speaking: boolean) {
-      setState((prevState) => ({ ...prevState, speaking }));
-    }
-
-    function onMuteStateChanged(audioMuted: boolean, videoMuted: boolean) {
-      setState((prevState) => ({ ...prevState, audioMuted, videoMuted }));
-    }
-
-    function onLocalVolumeChanged(localVolume: number) {
-      setState((prevState) => ({ ...prevState, localVolume }));
-    }
-
     function onUpdateCallFeed() {
       setState(getCallFeedState(callFeed));
     }
 
+    onUpdateCallFeed();
+
     if (callFeed) {
+      const onSpeaking = (speaking: boolean) => {
+        otelGroupCallMembership?.onSpeaking(
+          callFeed.getMember()!,
+          callFeed.deviceId!,
+          speaking
+        );
+        setState((prevState) => ({ ...prevState, speaking }));
+      };
+
+      const onMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => {
+        setState((prevState) => ({ ...prevState, audioMuted, videoMuted }));
+      };
+
+      const onLocalVolumeChanged = (localVolume: number) => {
+        setState((prevState) => ({ ...prevState, localVolume }));
+      };
+
       callFeed.on(CallFeedEvent.Speaking, onSpeaking);
       callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
       callFeed.on(CallFeedEvent.LocalVolumeChanged, onLocalVolumeChanged);
       callFeed.on(CallFeedEvent.NewStream, onUpdateCallFeed);
       callFeed.on(CallFeedEvent.Disposed, onUpdateCallFeed);
-    }
 
-    onUpdateCallFeed();
-
-    return () => {
-      if (callFeed) {
+      return () => {
         callFeed.removeListener(CallFeedEvent.Speaking, onSpeaking);
         callFeed.removeListener(
           CallFeedEvent.MuteStateChanged,
@@ -90,9 +98,9 @@ export function useCallFeed(callFeed: CallFeed | undefined): CallFeedState {
           onLocalVolumeChanged
         );
         callFeed.removeListener(CallFeedEvent.NewStream, onUpdateCallFeed);
-      }
-    };
-  }, [callFeed]);
+      };
+    }
+  }, [callFeed, otelGroupCallMembership]);
 
   return state;
 }