Merge pull request #971 from robintown/audio-observability

End-to-end audio observability

src/otel/OTelGroupCallMembership.ts

@@ -101,6 +101,7 @@ export class OTelGroupCallMembership {
     span: Span | undefined;
     stats: OTelStatsReportEvent[];
   };
+  private readonly speakingSpans = new Map<RoomMember, Map<string, Span>>();
 
   constructor(private groupCall: GroupCall, client: MatrixClient) {
     const clientId = client.getUserId();
@@ -125,6 +126,10 @@ export class OTelGroupCallMembership {
 
   public onJoinCall() {
     if (!ElementCallOpenTelemetry.instance) return;
+    if (this.callMembershipSpan !== undefined) {
+      logger.warn("Call membership span is already started");
+      return;
+    }
 
     // Create the main span that tracks the time we intend to be in the call
     this.callMembershipSpan =
@@ -151,10 +156,16 @@ export class OTelGroupCallMembership {
   }
 
   public onLeaveCall() {
-    this.callMembershipSpan?.addEvent("matrix.leaveCall");
+    if (this.callMembershipSpan === undefined) {
+      logger.warn("Call membership span is already ended");
+      return;
+    }
 
-    // and end the main span to indicate we've left
+    this.callMembershipSpan.addEvent("matrix.leaveCall");
-    if (this.callMembershipSpan) this.callMembershipSpan.end();
+    // and end the span to indicate we've left
+    this.callMembershipSpan.end();
+    this.callMembershipSpan = undefined;
+    this.groupCallContext = undefined;
   }
 
   public onUpdateRoomState(event: MatrixEvent) {
@@ -302,6 +313,36 @@ export class OTelGroupCallMembership {
     });
   }
 
+  public onSpeaking(member: RoomMember, deviceId: string, speaking: boolean) {
+    if (speaking) {
+      // Ensure that there's an audio activity span for this speaker
+      let deviceMap = this.speakingSpans.get(member);
+      if (deviceMap === undefined) {
+        deviceMap = new Map();
+        this.speakingSpans.set(member, deviceMap);
+      }
+
+      if (!deviceMap.has(deviceId)) {
+        const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
+          "matrix.audioActivity",
+          undefined,
+          this.groupCallContext
+        );
+        span.setAttribute("matrix.userId", member.userId);
+        span.setAttribute("matrix.displayName", member.rawDisplayName);
+
+        deviceMap.set(deviceId, span);
+      }
+    } else {
+      // End the audio activity span for this speaker, if any
+      const deviceMap = this.speakingSpans.get(member);
+      deviceMap?.get(deviceId)?.end();
+      deviceMap?.delete(deviceId);
+
+      if (deviceMap?.size === 0) this.speakingSpans.delete(member);
+    }
+  }
+
   public onCallError(error: CallError, call: MatrixCall) {
     const callTrackingInfo = this.callsByCallId.get(call.callId);
     if (!callTrackingInfo) {

src/room/InCallView.tsx

@@ -360,11 +360,11 @@ export function InCallView({
   const audioElements: JSX.Element[] = [];
   if (!spatialAudio || maximisedParticipant) {
     for (const item of items) {
-      if (item.isLocal) continue; // We don't want to render own audio
       audioElements.push(
         <AudioSink
           tileDescriptor={item}
           audioOutput={audioOutput}
+          otelGroupCallMembership={otelGroupCallMembership}
           key={item.id}
         />
       );

src/video-grid/AudioSink.tsx

@@ -16,6 +16,7 @@ limitations under the License.
 
 import React from "react";
 
+import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
 import { TileDescriptor } from "./TileDescriptor";
 import { useCallFeed } from "./useCallFeed";
 import { useMediaStream } from "./useMediaStream";
@@ -23,6 +24,7 @@ import { useMediaStream } from "./useMediaStream";
 interface Props {
   tileDescriptor: TileDescriptor;
   audioOutput: string;
+  otelGroupCallMembership?: OTelGroupCallMembership;
 }
 
 // Renders and <audio> element on the page playing the given stream
@@ -30,8 +32,12 @@ interface Props {
 export const AudioSink: React.FC<Props> = ({
   tileDescriptor,
   audioOutput,
+  otelGroupCallMembership,
 }: Props) => {
-  const { localVolume, stream } = useCallFeed(tileDescriptor.callFeed);
+  const { localVolume, stream } = useCallFeed(
+    tileDescriptor.callFeed,
+    otelGroupCallMembership
+  );
 
   const audioElementRef = useMediaStream(
     stream,

src/video-grid/useCallFeed.ts

@@ -18,6 +18,8 @@ import { useState, useEffect } from "react";
 import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed";
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
 
+import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
+
 interface CallFeedState {
   callFeed: CallFeed | undefined;
   isLocal: boolean;
@@ -46,40 +48,46 @@ function getCallFeedState(callFeed: CallFeed | undefined): CallFeedState {
   };
 }
 
-export function useCallFeed(callFeed: CallFeed | undefined): CallFeedState {
+export function useCallFeed(
+  callFeed: CallFeed | undefined,
+  otelGroupCallMembership?: OTelGroupCallMembership
+): CallFeedState {
   const [state, setState] = useState<CallFeedState>(() =>
     getCallFeedState(callFeed)
   );
 
   useEffect(() => {
-    function onSpeaking(speaking: boolean) {
-      setState((prevState) => ({ ...prevState, speaking }));
-    }
-
-    function onMuteStateChanged(audioMuted: boolean, videoMuted: boolean) {
-      setState((prevState) => ({ ...prevState, audioMuted, videoMuted }));
-    }
-
-    function onLocalVolumeChanged(localVolume: number) {
-      setState((prevState) => ({ ...prevState, localVolume }));
-    }
-
     function onUpdateCallFeed() {
       setState(getCallFeedState(callFeed));
     }
 
+    onUpdateCallFeed();
+
     if (callFeed) {
+      const onSpeaking = (speaking: boolean) => {
+        otelGroupCallMembership?.onSpeaking(
+          callFeed.getMember()!,
+          callFeed.deviceId!,
+          speaking
+        );
+        setState((prevState) => ({ ...prevState, speaking }));
+      };
+
+      const onMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => {
+        setState((prevState) => ({ ...prevState, audioMuted, videoMuted }));
+      };
+
+      const onLocalVolumeChanged = (localVolume: number) => {
+        setState((prevState) => ({ ...prevState, localVolume }));
+      };
+
       callFeed.on(CallFeedEvent.Speaking, onSpeaking);
       callFeed.on(CallFeedEvent.MuteStateChanged, onMuteStateChanged);
       callFeed.on(CallFeedEvent.LocalVolumeChanged, onLocalVolumeChanged);
       callFeed.on(CallFeedEvent.NewStream, onUpdateCallFeed);
       callFeed.on(CallFeedEvent.Disposed, onUpdateCallFeed);
-    }
 
-    onUpdateCallFeed();
+      return () => {
-
-    return () => {
-      if (callFeed) {
         callFeed.removeListener(CallFeedEvent.Speaking, onSpeaking);
         callFeed.removeListener(
           CallFeedEvent.MuteStateChanged,
@@ -90,9 +98,9 @@ export function useCallFeed(callFeed: CallFeed | undefined): CallFeedState {
           onLocalVolumeChanged
         );
         callFeed.removeListener(CallFeedEvent.NewStream, onUpdateCallFeed);
-      }
+      };
-    };
+    }
-  }, [callFeed]);
+  }, [callFeed, otelGroupCallMembership]);
 
   return state;
 }