Display active tracks in OTel metrics (#1085)

* Add track, feed and transceiver spans under call span

package.json

@@ -53,7 +53,7 @@
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e70a1a1effe59e6754f9a10cc2df8eef81638c7d",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1",
     "matrix-widget-api": "^1.3.1",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",

src/otel/OTelCall.ts

@@ -17,13 +17,33 @@ limitations under the License.
 import { Span } from "@opentelemetry/api";
 import { MatrixCall } from "matrix-js-sdk";
 import { CallEvent } from "matrix-js-sdk/src/webrtc/call";
+import {
+  TransceiverStats,
+  CallFeedStats,
+} from "matrix-js-sdk/src/webrtc/stats/statsReport";
 
 import { ObjectFlattener } from "./ObjectFlattener";
+import { ElementCallOpenTelemetry } from "./otel";
+import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
+import { OTelCallTransceiverMediaStreamSpan } from "./OTelCallTransceiverMediaStreamSpan";
+import { OTelCallFeedMediaStreamSpan } from "./OTelCallFeedMediaStreamSpan";
+
+type StreamId = string;
+type MID = string;
 
 /**
  * Tracks an individual call within a group call, either to a full-mesh peer or a focus
  */
 export class OTelCall {
+  private readonly trackFeedSpan = new Map<
+    StreamId,
+    OTelCallAbstractMediaStreamSpan
+  >();
+  private readonly trackTransceiverSpan = new Map<
+    MID,
+    OTelCallAbstractMediaStreamSpan
+  >();
+
   constructor(
     public userId: string,
     public deviceId: string,
@@ -116,4 +136,62 @@ export class OTelCall {
 
     this.span.addEvent("matrix.call.iceCandidateError", flatObject);
   };
+
+  public onCallFeedStats(callFeeds: CallFeedStats[]): void {
+    let prvFeeds: StreamId[] = [...this.trackFeedSpan.keys()];
+
+    callFeeds.forEach((feed) => {
+      if (!this.trackFeedSpan.has(feed.stream)) {
+        this.trackFeedSpan.set(
+          feed.stream,
+          new OTelCallFeedMediaStreamSpan(
+            ElementCallOpenTelemetry.instance,
+            this.span,
+            feed
+          )
+        );
+      }
+      this.trackFeedSpan.get(feed.stream)?.update(feed);
+      prvFeeds = prvFeeds.filter((prvStreamId) => prvStreamId !== feed.stream);
+    });
+
+    prvFeeds.forEach((prvStreamId) => {
+      this.trackFeedSpan.get(prvStreamId)?.end();
+      this.trackFeedSpan.delete(prvStreamId);
+    });
+  }
+
+  public onTransceiverStats(transceiverStats: TransceiverStats[]): void {
+    let prvTransSpan: MID[] = [...this.trackTransceiverSpan.keys()];
+
+    transceiverStats.forEach((transStats) => {
+      if (!this.trackTransceiverSpan.has(transStats.mid)) {
+        this.trackTransceiverSpan.set(
+          transStats.mid,
+          new OTelCallTransceiverMediaStreamSpan(
+            ElementCallOpenTelemetry.instance,
+            this.span,
+            transStats
+          )
+        );
+      }
+      this.trackTransceiverSpan.get(transStats.mid)?.update(transStats);
+      prvTransSpan = prvTransSpan.filter(
+        (prvStreamId) => prvStreamId !== transStats.mid
+      );
+    });
+
+    prvTransSpan.forEach((prvMID) => {
+      this.trackTransceiverSpan.get(prvMID)?.end();
+      this.trackTransceiverSpan.delete(prvMID);
+    });
+  }
+
+  public end(): void {
+    this.trackFeedSpan.forEach((feedSpan) => feedSpan.end());
+    this.trackTransceiverSpan.forEach((transceiverSpan) =>
+      transceiverSpan.end()
+    );
+    this.span.end();
+  }
 }

src/otel/OTelCallAbstractMediaStreamSpan.ts

@@ -0,0 +1,62 @@
+import opentelemetry, { Span } from "@opentelemetry/api";
+import { TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport";
+
+import { ElementCallOpenTelemetry } from "./otel";
+import { OTelCallMediaStreamTrackSpan } from "./OTelCallMediaStreamTrackSpan";
+
+type TrackId = string;
+
+export abstract class OTelCallAbstractMediaStreamSpan {
+  protected readonly trackSpans = new Map<
+    TrackId,
+    OTelCallMediaStreamTrackSpan
+  >();
+  public readonly span;
+
+  public constructor(
+    readonly oTel: ElementCallOpenTelemetry,
+    readonly callSpan: Span,
+    protected readonly type: string
+  ) {
+    const ctx = opentelemetry.trace.setSpan(
+      opentelemetry.context.active(),
+      callSpan
+    );
+    const options = {
+      links: [
+        {
+          context: callSpan.spanContext(),
+        },
+      ],
+    };
+    this.span = oTel.tracer.startSpan(this.type, options, ctx);
+  }
+
+  protected upsertTrackSpans(tracks: TrackStats[]) {
+    let prvTracks: TrackId[] = [...this.trackSpans.keys()];
+    tracks.forEach((t) => {
+      if (!this.trackSpans.has(t.id)) {
+        this.trackSpans.set(
+          t.id,
+          new OTelCallMediaStreamTrackSpan(this.oTel, this.span, t)
+        );
+      }
+      this.trackSpans.get(t.id)?.update(t);
+      prvTracks = prvTracks.filter((prvTrackId) => prvTrackId !== t.id);
+    });
+
+    prvTracks.forEach((prvTrackId) => {
+      this.trackSpans.get(prvTrackId)?.end();
+      this.trackSpans.delete(prvTrackId);
+    });
+  }
+
+  public abstract update(data: Object): void;
+
+  public end(): void {
+    this.trackSpans.forEach((tSpan) => {
+      tSpan.end();
+    });
+    this.span.end();
+  }
+}

src/otel/OTelCallFeedMediaStreamSpan.ts

@@ -0,0 +1,57 @@
+import { Span } from "@opentelemetry/api";
+import {
+  CallFeedStats,
+  TrackStats,
+} from "matrix-js-sdk/src/webrtc/stats/statsReport";
+
+import { ElementCallOpenTelemetry } from "./otel";
+import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
+
+export class OTelCallFeedMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
+  private readonly prev: { isAudioMuted: boolean; isVideoMuted: boolean };
+
+  constructor(
+    readonly oTel: ElementCallOpenTelemetry,
+    readonly callSpan: Span,
+    callFeed: CallFeedStats
+  ) {
+    const postFix =
+      callFeed.type === "local" && callFeed.prefix === "from-call-feed"
+        ? "(clone)"
+        : "";
+    super(oTel, callSpan, `matrix.call.feed.${callFeed.type}${postFix}`);
+    this.span.setAttribute("feed.streamId", callFeed.stream);
+    this.span.setAttribute("feed.type", callFeed.type);
+    this.span.setAttribute("feed.readFrom", callFeed.prefix);
+    this.span.setAttribute("feed.purpose", callFeed.purpose);
+    this.prev = {
+      isAudioMuted: callFeed.isAudioMuted,
+      isVideoMuted: callFeed.isVideoMuted,
+    };
+    this.span.addEvent("matrix.call.feed.initState", this.prev);
+  }
+
+  public update(callFeed: CallFeedStats): void {
+    if (this.prev.isAudioMuted !== callFeed.isAudioMuted) {
+      this.span.addEvent("matrix.call.feed.audioMuted", {
+        isAudioMuted: callFeed.isAudioMuted,
+      });
+      this.prev.isAudioMuted = callFeed.isAudioMuted;
+    }
+    if (this.prev.isVideoMuted !== callFeed.isVideoMuted) {
+      this.span.addEvent("matrix.call.feed.isVideoMuted", {
+        isVideoMuted: callFeed.isVideoMuted,
+      });
+      this.prev.isVideoMuted = callFeed.isVideoMuted;
+    }
+
+    const trackStats: TrackStats[] = [];
+    if (callFeed.video) {
+      trackStats.push(callFeed.video);
+    }
+    if (callFeed.audio) {
+      trackStats.push(callFeed.audio);
+    }
+    this.upsertTrackSpans(trackStats);
+  }
+}

src/otel/OTelCallMediaStreamTrackSpan.ts

@@ -0,0 +1,62 @@
+import { TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport";
+import opentelemetry, { Span } from "@opentelemetry/api";
+
+import { ElementCallOpenTelemetry } from "./otel";
+
+export class OTelCallMediaStreamTrackSpan {
+  private readonly span: Span;
+  private prev: TrackStats;
+
+  public constructor(
+    readonly oTel: ElementCallOpenTelemetry,
+    readonly streamSpan: Span,
+    data: TrackStats
+  ) {
+    const ctx = opentelemetry.trace.setSpan(
+      opentelemetry.context.active(),
+      streamSpan
+    );
+    const options = {
+      links: [
+        {
+          context: streamSpan.spanContext(),
+        },
+      ],
+    };
+    const type = `matrix.call.track.${data.label}.${data.kind}`;
+    this.span = oTel.tracer.startSpan(type, options, ctx);
+    this.span.setAttribute("track.trackId", data.id);
+    this.span.setAttribute("track.kind", data.kind);
+    this.span.setAttribute("track.constrainDeviceId", data.constrainDeviceId);
+    this.span.setAttribute("track.settingDeviceId", data.settingDeviceId);
+    this.span.setAttribute("track.label", data.label);
+
+    this.span.addEvent("matrix.call.track.initState", {
+      readyState: data.readyState,
+      muted: data.muted,
+      enabled: data.enabled,
+    });
+    this.prev = data;
+  }
+
+  public update(data: TrackStats): void {
+    if (this.prev.muted !== data.muted) {
+      this.span.addEvent("matrix.call.track.muted", { muted: data.muted });
+    }
+    if (this.prev.enabled !== data.enabled) {
+      this.span.addEvent("matrix.call.track.enabled", {
+        enabled: data.enabled,
+      });
+    }
+    if (this.prev.readyState !== data.readyState) {
+      this.span.addEvent("matrix.call.track.readyState", {
+        readyState: data.readyState,
+      });
+    }
+    this.prev = data;
+  }
+
+  public end(): void {
+    this.span.end();
+  }
+}

src/otel/OTelCallTransceiverMediaStreamSpan.ts

@@ -0,0 +1,54 @@
+import { Span } from "@opentelemetry/api";
+import {
+  TrackStats,
+  TransceiverStats,
+} from "matrix-js-sdk/src/webrtc/stats/statsReport";
+
+import { ElementCallOpenTelemetry } from "./otel";
+import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan";
+
+export class OTelCallTransceiverMediaStreamSpan extends OTelCallAbstractMediaStreamSpan {
+  private readonly prev: {
+    direction: string;
+    currentDirection: string;
+  };
+
+  constructor(
+    readonly oTel: ElementCallOpenTelemetry,
+    readonly callSpan: Span,
+    stats: TransceiverStats
+  ) {
+    super(oTel, callSpan, `matrix.call.transceiver.${stats.mid}`);
+    this.span.setAttribute("transceiver.mid", stats.mid);
+
+    this.prev = {
+      direction: stats.direction,
+      currentDirection: stats.currentDirection,
+    };
+    this.span.addEvent("matrix.call.transceiver.initState", this.prev);
+  }
+
+  public update(stats: TransceiverStats): void {
+    if (this.prev.currentDirection !== stats.currentDirection) {
+      this.span.addEvent("matrix.call.transceiver.currentDirection", {
+        currentDirection: stats.currentDirection,
+      });
+      this.prev.currentDirection = stats.currentDirection;
+    }
+    if (this.prev.direction !== stats.direction) {
+      this.span.addEvent("matrix.call.transceiver.direction", {
+        direction: stats.direction,
+      });
+      this.prev.direction = stats.direction;
+    }
+
+    const trackStats: TrackStats[] = [];
+    if (stats.sender) {
+      trackStats.push(stats.sender);
+    }
+    if (stats.receiver) {
+      trackStats.push(stats.receiver);
+    }
+    this.upsertTrackSpans(trackStats);
+  }
+}

src/otel/OTelGroupCallMembership.ts

@@ -38,6 +38,7 @@ import {
   ConnectionStatsReport,
   ByteSentStatsReport,
   SummaryStatsReport,
+  CallFeedReport,
 } from "matrix-js-sdk/src/webrtc/stats/statsReport";
 import { setSpan } from "@opentelemetry/api/build/esm/trace/context-utils";
 
@@ -174,7 +175,7 @@ export class OTelGroupCallMembership {
         userCalls.get(callTrackingInfo.deviceId).callId !==
           callTrackingInfo.call.callId
       ) {
-        callTrackingInfo.span.end();
+        callTrackingInfo.end();
         this.callsByCallId.delete(callTrackingInfo.call.callId);
       }
     }
@@ -330,6 +331,35 @@ export class OTelGroupCallMembership {
     });
   }
 
+  public onCallFeedStatsReport(report: GroupCallStatsReport<CallFeedReport>) {
+    if (!ElementCallOpenTelemetry.instance) return;
+    let call: OTelCall | undefined;
+    const callId = report.report?.callId;
+
+    if (callId) {
+      call = this.callsByCallId.get(callId);
+    }
+
+    if (!call) {
+      this.callMembershipSpan?.addEvent(
+        OTelStatsReportType.CallFeedReport + "_unknown_callId",
+        {
+          "call.callId": callId,
+          "call.opponentMemberId": report.report?.opponentMemberId
+            ? report.report?.opponentMemberId
+            : "unknown",
+        }
+      );
+      logger.error(
+        `Received ${OTelStatsReportType.CallFeedReport} with unknown call ID: ${callId}`
+      );
+      return;
+    } else {
+      call.onCallFeedStats(report.report.callFeeds);
+      call.onTransceiverStats(report.report.transceiver);
+    }
+  }
+
   public onConnectionStatsReport(
     statsReport: GroupCallStatsReport<ConnectionStatsReport>
   ) {
@@ -440,4 +470,5 @@ enum OTelStatsReportType {
   ConnectionReport = "matrix.call.stats.connection",
   ByteSentReport = "matrix.call.stats.byteSent",
   SummaryReport = "matrix.stats.summary",
+  CallFeedReport = "matrix.stats.call_feed",
 }

src/room/useGroupCall.ts

@@ -34,6 +34,7 @@ import {
   ByteSentStatsReport,
   ConnectionStatsReport,
   SummaryStatsReport,
+  CallFeedReport,
 } from "matrix-js-sdk/src/webrtc/stats/statsReport";
 
 import { usePageUnload } from "./usePageUnload";
@@ -363,6 +364,12 @@ export function useGroupCall(
       groupCallOTelMembership?.onSummaryStatsReport(report);
     }
 
+    function onCallFeedStatsReport(
+      report: GroupCallStatsReport<CallFeedReport>
+    ): void {
+      groupCallOTelMembership?.onCallFeedStatsReport(report);
+    }
+
     groupCall.on(GroupCallEvent.GroupCallStateChanged, onGroupCallStateChanged);
     groupCall.on(GroupCallEvent.UserMediaFeedsChanged, onUserMediaFeedsChanged);
     groupCall.on(
@@ -387,6 +394,11 @@ export function useGroupCall(
       onByteSentStatsReport
     );
     groupCall.on(GroupCallStatsReportEvent.SummaryStats, onSummaryStatsReport);
+    groupCall.on(
+      GroupCallStatsReportEvent.CallFeedStats,
+      onCallFeedStatsReport
+    );
+
     groupCall.room.currentState.on(
       RoomStateEvent.Update,
       checkForParallelCalls
@@ -450,6 +462,10 @@ export function useGroupCall(
         GroupCallStatsReportEvent.SummaryStats,
         onSummaryStatsReport
       );
+      groupCall.removeListener(
+        GroupCallStatsReportEvent.CallFeedStats,
+        onCallFeedStatsReport
+      );
       groupCall.room.currentState.off(
         RoomStateEvent.Update,
         checkForParallelCalls

yarn.lock

@@ -10557,9 +10557,9 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#e70a1a1effe59e6754f9a10cc2df8eef81638c7d":
-  version "25.1.1"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/e70a1a1effe59e6754f9a10cc2df8eef81638c7d"
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1":
+  version "26.0.0"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1"
   dependencies:
     "@babel/runtime" "^7.12.5"
     "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9"