diff --git a/package.json b/package.json index 0d5325b..9923b6b 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "i18next-browser-languagedetector": "^6.1.8", "i18next-http-backend": "^1.4.4", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e70a1a1effe59e6754f9a10cc2df8eef81638c7d", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/src/otel/OTelCall.ts b/src/otel/OTelCall.ts index 79cc38d..eae1f34 100644 --- a/src/otel/OTelCall.ts +++ b/src/otel/OTelCall.ts @@ -17,13 +17,33 @@ limitations under the License. import { Span } from "@opentelemetry/api"; import { MatrixCall } from "matrix-js-sdk"; import { CallEvent } from "matrix-js-sdk/src/webrtc/call"; +import { + TransceiverStats, + CallFeedStats, +} from "matrix-js-sdk/src/webrtc/stats/statsReport"; import { ObjectFlattener } from "./ObjectFlattener"; +import { ElementCallOpenTelemetry } from "./otel"; +import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan"; +import { OTelCallTransceiverMediaStreamSpan } from "./OTelCallTransceiverMediaStreamSpan"; +import { OTelCallFeedMediaStreamSpan } from "./OTelCallFeedMediaStreamSpan"; + +type StreamId = string; +type MID = string; /** * Tracks an individual call within a group call, either to a full-mesh peer or a focus */ export class OTelCall { + private readonly trackFeedSpan = new Map< + StreamId, + OTelCallAbstractMediaStreamSpan + >(); + private readonly trackTransceiverSpan = new Map< + MID, + OTelCallAbstractMediaStreamSpan + >(); + constructor( public userId: string, public deviceId: string, @@ -116,4 +136,62 @@ export class OTelCall { this.span.addEvent("matrix.call.iceCandidateError", flatObject); }; + + public onCallFeedStats(callFeeds: CallFeedStats[]): void { + let prvFeeds: StreamId[] = [...this.trackFeedSpan.keys()]; + + callFeeds.forEach((feed) => { + if (!this.trackFeedSpan.has(feed.stream)) { + this.trackFeedSpan.set( + feed.stream, + new OTelCallFeedMediaStreamSpan( + ElementCallOpenTelemetry.instance, + this.span, + feed + ) + ); + } + this.trackFeedSpan.get(feed.stream)?.update(feed); + prvFeeds = prvFeeds.filter((prvStreamId) => prvStreamId !== feed.stream); + }); + + prvFeeds.forEach((prvStreamId) => { + this.trackFeedSpan.get(prvStreamId)?.end(); + this.trackFeedSpan.delete(prvStreamId); + }); + } + + public onTransceiverStats(transceiverStats: TransceiverStats[]): void { + let prvTransSpan: MID[] = [...this.trackTransceiverSpan.keys()]; + + transceiverStats.forEach((transStats) => { + if (!this.trackTransceiverSpan.has(transStats.mid)) { + this.trackTransceiverSpan.set( + transStats.mid, + new OTelCallTransceiverMediaStreamSpan( + ElementCallOpenTelemetry.instance, + this.span, + transStats + ) + ); + } + this.trackTransceiverSpan.get(transStats.mid)?.update(transStats); + prvTransSpan = prvTransSpan.filter( + (prvStreamId) => prvStreamId !== transStats.mid + ); + }); + + prvTransSpan.forEach((prvMID) => { + this.trackTransceiverSpan.get(prvMID)?.end(); + this.trackTransceiverSpan.delete(prvMID); + }); + } + + public end(): void { + this.trackFeedSpan.forEach((feedSpan) => feedSpan.end()); + this.trackTransceiverSpan.forEach((transceiverSpan) => + transceiverSpan.end() + ); + this.span.end(); + } } diff --git a/src/otel/OTelCallAbstractMediaStreamSpan.ts b/src/otel/OTelCallAbstractMediaStreamSpan.ts new file mode 100644 index 0000000..aa77051 --- /dev/null +++ b/src/otel/OTelCallAbstractMediaStreamSpan.ts @@ -0,0 +1,62 @@ +import opentelemetry, { Span } from "@opentelemetry/api"; +import { TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport"; + +import { ElementCallOpenTelemetry } from "./otel"; +import { OTelCallMediaStreamTrackSpan } from "./OTelCallMediaStreamTrackSpan"; + +type TrackId = string; + +export abstract class OTelCallAbstractMediaStreamSpan { + protected readonly trackSpans = new Map< + TrackId, + OTelCallMediaStreamTrackSpan + >(); + public readonly span; + + public constructor( + readonly oTel: ElementCallOpenTelemetry, + readonly callSpan: Span, + protected readonly type: string + ) { + const ctx = opentelemetry.trace.setSpan( + opentelemetry.context.active(), + callSpan + ); + const options = { + links: [ + { + context: callSpan.spanContext(), + }, + ], + }; + this.span = oTel.tracer.startSpan(this.type, options, ctx); + } + + protected upsertTrackSpans(tracks: TrackStats[]) { + let prvTracks: TrackId[] = [...this.trackSpans.keys()]; + tracks.forEach((t) => { + if (!this.trackSpans.has(t.id)) { + this.trackSpans.set( + t.id, + new OTelCallMediaStreamTrackSpan(this.oTel, this.span, t) + ); + } + this.trackSpans.get(t.id)?.update(t); + prvTracks = prvTracks.filter((prvTrackId) => prvTrackId !== t.id); + }); + + prvTracks.forEach((prvTrackId) => { + this.trackSpans.get(prvTrackId)?.end(); + this.trackSpans.delete(prvTrackId); + }); + } + + public abstract update(data: Object): void; + + public end(): void { + this.trackSpans.forEach((tSpan) => { + tSpan.end(); + }); + this.span.end(); + } +} diff --git a/src/otel/OTelCallFeedMediaStreamSpan.ts b/src/otel/OTelCallFeedMediaStreamSpan.ts new file mode 100644 index 0000000..6023fa6 --- /dev/null +++ b/src/otel/OTelCallFeedMediaStreamSpan.ts @@ -0,0 +1,57 @@ +import { Span } from "@opentelemetry/api"; +import { + CallFeedStats, + TrackStats, +} from "matrix-js-sdk/src/webrtc/stats/statsReport"; + +import { ElementCallOpenTelemetry } from "./otel"; +import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan"; + +export class OTelCallFeedMediaStreamSpan extends OTelCallAbstractMediaStreamSpan { + private readonly prev: { isAudioMuted: boolean; isVideoMuted: boolean }; + + constructor( + readonly oTel: ElementCallOpenTelemetry, + readonly callSpan: Span, + callFeed: CallFeedStats + ) { + const postFix = + callFeed.type === "local" && callFeed.prefix === "from-call-feed" + ? "(clone)" + : ""; + super(oTel, callSpan, `matrix.call.feed.${callFeed.type}${postFix}`); + this.span.setAttribute("feed.streamId", callFeed.stream); + this.span.setAttribute("feed.type", callFeed.type); + this.span.setAttribute("feed.readFrom", callFeed.prefix); + this.span.setAttribute("feed.purpose", callFeed.purpose); + this.prev = { + isAudioMuted: callFeed.isAudioMuted, + isVideoMuted: callFeed.isVideoMuted, + }; + this.span.addEvent("matrix.call.feed.initState", this.prev); + } + + public update(callFeed: CallFeedStats): void { + if (this.prev.isAudioMuted !== callFeed.isAudioMuted) { + this.span.addEvent("matrix.call.feed.audioMuted", { + isAudioMuted: callFeed.isAudioMuted, + }); + this.prev.isAudioMuted = callFeed.isAudioMuted; + } + if (this.prev.isVideoMuted !== callFeed.isVideoMuted) { + this.span.addEvent("matrix.call.feed.isVideoMuted", { + isVideoMuted: callFeed.isVideoMuted, + }); + this.prev.isVideoMuted = callFeed.isVideoMuted; + } + + const trackStats: TrackStats[] = []; + if (callFeed.video) { + trackStats.push(callFeed.video); + } + if (callFeed.audio) { + trackStats.push(callFeed.audio); + } + this.upsertTrackSpans(trackStats); + } +} diff --git a/src/otel/OTelCallMediaStreamTrackSpan.ts b/src/otel/OTelCallMediaStreamTrackSpan.ts new file mode 100644 index 0000000..935e22f --- /dev/null +++ b/src/otel/OTelCallMediaStreamTrackSpan.ts @@ -0,0 +1,62 @@ +import { TrackStats } from "matrix-js-sdk/src/webrtc/stats/statsReport"; +import opentelemetry, { Span } from "@opentelemetry/api"; + +import { ElementCallOpenTelemetry } from "./otel"; + +export class OTelCallMediaStreamTrackSpan { + private readonly span: Span; + private prev: TrackStats; + + public constructor( + readonly oTel: ElementCallOpenTelemetry, + readonly streamSpan: Span, + data: TrackStats + ) { + const ctx = opentelemetry.trace.setSpan( + opentelemetry.context.active(), + streamSpan + ); + const options = { + links: [ + { + context: streamSpan.spanContext(), + }, + ], + }; + const type = `matrix.call.track.${data.label}.${data.kind}`; + this.span = oTel.tracer.startSpan(type, options, ctx); + this.span.setAttribute("track.trackId", data.id); + this.span.setAttribute("track.kind", data.kind); + this.span.setAttribute("track.constrainDeviceId", data.constrainDeviceId); + this.span.setAttribute("track.settingDeviceId", data.settingDeviceId); + this.span.setAttribute("track.label", data.label); + + this.span.addEvent("matrix.call.track.initState", { + readyState: data.readyState, + muted: data.muted, + enabled: data.enabled, + }); + this.prev = data; + } + + public update(data: TrackStats): void { + if (this.prev.muted !== data.muted) { + this.span.addEvent("matrix.call.track.muted", { muted: data.muted }); + } + if (this.prev.enabled !== data.enabled) { + this.span.addEvent("matrix.call.track.enabled", { + enabled: data.enabled, + }); + } + if (this.prev.readyState !== data.readyState) { + this.span.addEvent("matrix.call.track.readyState", { + readyState: data.readyState, + }); + } + this.prev = data; + } + + public end(): void { + this.span.end(); + } +} diff --git a/src/otel/OTelCallTransceiverMediaStreamSpan.ts b/src/otel/OTelCallTransceiverMediaStreamSpan.ts new file mode 100644 index 0000000..97006cd --- /dev/null +++ b/src/otel/OTelCallTransceiverMediaStreamSpan.ts @@ -0,0 +1,54 @@ +import { Span } from "@opentelemetry/api"; +import { + TrackStats, + TransceiverStats, +} from "matrix-js-sdk/src/webrtc/stats/statsReport"; + +import { ElementCallOpenTelemetry } from "./otel"; +import { OTelCallAbstractMediaStreamSpan } from "./OTelCallAbstractMediaStreamSpan"; + +export class OTelCallTransceiverMediaStreamSpan extends OTelCallAbstractMediaStreamSpan { + private readonly prev: { + direction: string; + currentDirection: string; + }; + + constructor( + readonly oTel: ElementCallOpenTelemetry, + readonly callSpan: Span, + stats: TransceiverStats + ) { + super(oTel, callSpan, `matrix.call.transceiver.${stats.mid}`); + this.span.setAttribute("transceiver.mid", stats.mid); + + this.prev = { + direction: stats.direction, + currentDirection: stats.currentDirection, + }; + this.span.addEvent("matrix.call.transceiver.initState", this.prev); + } + + public update(stats: TransceiverStats): void { + if (this.prev.currentDirection !== stats.currentDirection) { + this.span.addEvent("matrix.call.transceiver.currentDirection", { + currentDirection: stats.currentDirection, + }); + this.prev.currentDirection = stats.currentDirection; + } + if (this.prev.direction !== stats.direction) { + this.span.addEvent("matrix.call.transceiver.direction", { + direction: stats.direction, + }); + this.prev.direction = stats.direction; + } + + const trackStats: TrackStats[] = []; + if (stats.sender) { + trackStats.push(stats.sender); + } + if (stats.receiver) { + trackStats.push(stats.receiver); + } + this.upsertTrackSpans(trackStats); + } +} diff --git a/src/otel/OTelGroupCallMembership.ts b/src/otel/OTelGroupCallMembership.ts index a8d93b6..fa1e164 100644 --- a/src/otel/OTelGroupCallMembership.ts +++ b/src/otel/OTelGroupCallMembership.ts @@ -38,6 +38,7 @@ import { ConnectionStatsReport, ByteSentStatsReport, SummaryStatsReport, + CallFeedReport, } from "matrix-js-sdk/src/webrtc/stats/statsReport"; import { setSpan } from "@opentelemetry/api/build/esm/trace/context-utils"; @@ -174,7 +175,7 @@ export class OTelGroupCallMembership { userCalls.get(callTrackingInfo.deviceId).callId !== callTrackingInfo.call.callId ) { - callTrackingInfo.span.end(); + callTrackingInfo.end(); this.callsByCallId.delete(callTrackingInfo.call.callId); } } @@ -330,6 +331,35 @@ export class OTelGroupCallMembership { }); } + public onCallFeedStatsReport(report: GroupCallStatsReport) { + if (!ElementCallOpenTelemetry.instance) return; + let call: OTelCall | undefined; + const callId = report.report?.callId; + + if (callId) { + call = this.callsByCallId.get(callId); + } + + if (!call) { + this.callMembershipSpan?.addEvent( + OTelStatsReportType.CallFeedReport + "_unknown_callId", + { + "call.callId": callId, + "call.opponentMemberId": report.report?.opponentMemberId + ? report.report?.opponentMemberId + : "unknown", + } + ); + logger.error( + `Received ${OTelStatsReportType.CallFeedReport} with unknown call ID: ${callId}` + ); + return; + } else { + call.onCallFeedStats(report.report.callFeeds); + call.onTransceiverStats(report.report.transceiver); + } + } + public onConnectionStatsReport( statsReport: GroupCallStatsReport ) { @@ -440,4 +470,5 @@ enum OTelStatsReportType { ConnectionReport = "matrix.call.stats.connection", ByteSentReport = "matrix.call.stats.byteSent", SummaryReport = "matrix.stats.summary", + CallFeedReport = "matrix.stats.call_feed", } diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts index 0126e1c..153bb18 100644 --- a/src/room/useGroupCall.ts +++ b/src/room/useGroupCall.ts @@ -34,6 +34,7 @@ import { ByteSentStatsReport, ConnectionStatsReport, SummaryStatsReport, + CallFeedReport, } from "matrix-js-sdk/src/webrtc/stats/statsReport"; import { usePageUnload } from "./usePageUnload"; @@ -363,6 +364,12 @@ export function useGroupCall( groupCallOTelMembership?.onSummaryStatsReport(report); } + function onCallFeedStatsReport( + report: GroupCallStatsReport + ): void { + groupCallOTelMembership?.onCallFeedStatsReport(report); + } + groupCall.on(GroupCallEvent.GroupCallStateChanged, onGroupCallStateChanged); groupCall.on(GroupCallEvent.UserMediaFeedsChanged, onUserMediaFeedsChanged); groupCall.on( @@ -387,6 +394,11 @@ export function useGroupCall( onByteSentStatsReport ); groupCall.on(GroupCallStatsReportEvent.SummaryStats, onSummaryStatsReport); + groupCall.on( + GroupCallStatsReportEvent.CallFeedStats, + onCallFeedStatsReport + ); + groupCall.room.currentState.on( RoomStateEvent.Update, checkForParallelCalls @@ -450,6 +462,10 @@ export function useGroupCall( GroupCallStatsReportEvent.SummaryStats, onSummaryStatsReport ); + groupCall.removeListener( + GroupCallStatsReportEvent.CallFeedStats, + onCallFeedStatsReport + ); groupCall.room.currentState.off( RoomStateEvent.Update, checkForParallelCalls diff --git a/yarn.lock b/yarn.lock index c9c9a44..1b5602a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10557,9 +10557,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#e70a1a1effe59e6754f9a10cc2df8eef81638c7d": - version "25.1.1" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/e70a1a1effe59e6754f9a10cc2df8eef81638c7d" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1": + version "26.0.0" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9"