Merge branch 'dbkr/otel' into enricoschw/real-time-media-statistics-for-full-mesh

.github/workflows/publish.yaml

@@ -40,6 +40,9 @@ jobs:
           SENTRY_URL: ${{ secrets.SENTRY_URL }}
           SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
           VITE_APP_VERSION: ${{ github.event.release.tag_name }}
+          # This appears to be necessary to stop Vite from OOMing
+          # https://github.com/vitejs/vite/issues/2433
+          NODE_OPTIONS: "--max-old-space-size=16384"
 
       - name: Create Tarball
         env:

config/otel_dev/README.md

@@ -1,7 +1,7 @@
 # OpenTelemetry Collector for development
 
 This directory contains a docker compose file that starts a jaeger all-in-one instance
-with an in-memory database, along with a standalong OpenTelemetry collector that forwards
+with an in-memory database, along with a standalone OpenTelemetry collector that forwards
 traces into the jaeger. Jaeger has a built-in OpenTelemetry collector, but it can't be
 configured to send CORS headers so can't be used from a browser. This sets the config on
 the collector to send CORS headers.

config/otel_dev/collector-gateway.yaml

@@ -7,7 +7,8 @@ receivers:
           allowed_origins:
             # This can't be '*' because opentelemetry-js uses sendBeacon which always operates
             # in 'withCredentials' mode, which browsers don't allow with an allow-origin of '*'
-            - "http://*"
+            #- "https://pr976--element-call.netlify.app"
+            - "https://*"
           allowed_headers:
             - "*"
 processors:

package.json

@@ -53,8 +53,8 @@
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#2cd38e91eee1f5b16a9be0caba6ff19486b95f31",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#042f2ed76c501c10dde98a31732fd92d862e2187",
-    "matrix-widget-api": "^1.0.0",
+    "matrix-widget-api": "^1.3.1",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",
     "pako": "^2.0.4",

src/analytics/OtelPosthogExporter.ts

@@ -18,9 +18,10 @@ import { SpanExporter, ReadableSpan } from "@opentelemetry/sdk-trace-base";
 import { ExportResult, ExportResultCode } from "@opentelemetry/core";
 
 import { PosthogAnalytics } from "./PosthogAnalytics";
+
 /**
- * This is implementation of {@link SpanExporter} that prints spans to the
+ * This is implementation of {@link SpanExporter} that sends spans
- * console. This class can be used for diagnostic purposes.
+ * to Posthog
  */
 export class PosthogSpanExporter implements SpanExporter {
   /**

src/config/ConfigOptions.ts

@@ -37,7 +37,8 @@ export interface ConfigOptions {
   };
 
   /**
-   * Controls whether to to send OpenTelemetry debugging data to collector
+   * Sets the URL to send opentelemetry data to. If unset, opentelemetry will
+   * be disabled.
    */
   opentelemetry?: {
     collector_url: string;

src/otel/OTelGroupCallMembership.ts

@@ -14,22 +14,33 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import opentelemetry, { Span, Attributes } from "@opentelemetry/api";
+import opentelemetry, { Span, Attributes, Context } from "@opentelemetry/api";
-import { SemanticResourceAttributes } from "@opentelemetry/semantic-conventions";
 import {
   GroupCall,
   MatrixClient,
   MatrixEvent,
   RoomMember,
 } from "matrix-js-sdk";
-import { VoipEvent } from "matrix-js-sdk/src/webrtc/call";
+import { logger } from "matrix-js-sdk/src/logger";
-import { GroupCallStatsReport } from "matrix-js-sdk/src/webrtc/groupCall";
+import {
+  CallError,
+  CallState,
+  MatrixCall,
+  VoipEvent,
+} from "matrix-js-sdk/src/webrtc/call";
+import {
+  CallsByUserAndDevice,
+  GroupCallError,
+  GroupCallEvent,
+  GroupCallStatsReport,
+} from "matrix-js-sdk/src/webrtc/groupCall";
 import {
   ConnectionStatsReport,
   ByteSentStatsReport,
 } from "matrix-js-sdk/src/webrtc/stats/statsReport";
 import { setSpan } from "@opentelemetry/api/build/esm/trace/context-utils";
 
+
 import { ElementCallOpenTelemetry } from "./otel";
 import { ObjectFlattener } from "./ObjectFlattener";
 
@@ -75,13 +86,23 @@ function flattenVoipEventRecursive(
   }
 }
 
+interface CallTrackingInfo {
+  userId: string;
+  deviceId: string;
+  call: MatrixCall;
+  span: Span;
+}
+
 /**
  * Represent the span of time which we intend to be joined to a group call
  */
 export class OTelGroupCallMembership {
   private callMembershipSpan?: Span;
+  private groupCallContext?: Context;
   private myUserId = "unknown";
+  private myDeviceId: string;
   private myMember?: RoomMember;
+  private callsByCallId = new Map<string, CallTrackingInfo>();
   private statsReportSpan: {
     span: Span | undefined;
     stats: OTelStatsReportEvent[];
@@ -96,12 +117,16 @@ export class OTelGroupCallMembership {
         this.myMember = myMember;
       }
     }
-
+    this.myDeviceId = client.getDeviceId();
     this.statsReportSpan = { span: undefined, stats: [] };
+    this.groupCall.on(GroupCallEvent.CallsChanged, this.onCallsChanged);
+  }
 
-    ElementCallOpenTelemetry.instance.provider.resource.attributes[
+  dispose() {
-      SemanticResourceAttributes.SERVICE_NAME
+    this.groupCall.removeListener(
-    ] = `element-call-${this.myUserId}-${client.getDeviceId()}`;
+      GroupCallEvent.CallsChanged,
+      this.onCallsChanged
+    );
   }
 
   public onJoinCall() {
@@ -115,12 +140,13 @@ export class OTelGroupCallMembership {
       this.groupCall.groupCallId
     );
     this.callMembershipSpan.setAttribute("matrix.userId", this.myUserId);
+    this.callMembershipSpan.setAttribute("matrix.deviceId", this.myDeviceId);
     this.callMembershipSpan.setAttribute(
       "matrix.displayName",
       this.myMember ? this.myMember.name : "unknown-name"
     );
 
-    opentelemetry.trace.setSpan(
+    this.groupCallContext = opentelemetry.trace.setSpan(
       opentelemetry.context.active(),
       this.callMembershipSpan
     );
@@ -132,7 +158,7 @@ export class OTelGroupCallMembership {
     this.callMembershipSpan?.addEvent("matrix.leaveCall");
 
     // and end the main span to indicate we've left
-    this.callMembershipSpan?.end();
+    if (this.callMembershipSpan) this.callMembershipSpan.end();
   }
 
   public onUpdateRoomState(event: MatrixEvent) {
@@ -145,12 +171,55 @@ export class OTelGroupCallMembership {
     }
 
     this.callMembershipSpan?.addEvent(
-      `otel_onRoomStateEvent_${event.getType()}`,
+      `matrix.roomStateEvent_${event.getType()}`,
       flattenVoipEvent(event.getContent())
     );
   }
 
-  public onSendEvent(event: VoipEvent) {
+  public onCallsChanged = (calls: CallsByUserAndDevice) => {
+    for (const [userId, userCalls] of calls.entries()) {
+      for (const [deviceId, call] of userCalls.entries()) {
+        if (!this.callsByCallId.has(call.callId)) {
+          const span = ElementCallOpenTelemetry.instance.tracer.startSpan(
+            `matrix.call`,
+            undefined,
+            this.groupCallContext
+          );
+          // XXX: anonymity
+          span.setAttribute("matrix.call.target.userId", userId);
+          span.setAttribute("matrix.call.target.deviceId", deviceId);
+          this.callsByCallId.set(call.callId, {
+            userId,
+            deviceId,
+            call,
+            span,
+          });
+        }
+      }
+    }
+
+    for (const callTrackingInfo of this.callsByCallId.values()) {
+      const userCalls = calls.get(callTrackingInfo.userId);
+      if (!userCalls || !userCalls.has(callTrackingInfo.deviceId)) {
+        callTrackingInfo.span.end();
+        this.callsByCallId.delete(callTrackingInfo.call.callId);
+      }
+    }
+  };
+
+  public onCallStateChange(call: MatrixCall, newState: CallState) {
+    const callTrackingInfo = this.callsByCallId.get(call.callId);
+    if (!callTrackingInfo) {
+      logger.error(`Got call state change for unknown call ID ${call.callId}`);
+      return;
+    }
+
+    callTrackingInfo.span.addEvent("matrix.call.stateChange", {
+      state: newState,
+    });
+  }
+
+  public onSendEvent(call: MatrixCall, event: VoipEvent) {
     const eventType = event.eventType as string;
     if (!eventType.startsWith("m.call")) return;
 
@@ -167,6 +236,37 @@ export class OTelGroupCallMembership {
     }
   }
 
+  public onReceivedVoipEvent(event: MatrixEvent) {
+    // These come straight from CallEventHandler so don't have
+    // a call already associated (in principle we could receive
+    // events for calls we don't know about).
+    const callId = event.getContent().call_id;
+    if (!callId) {
+      this.callMembershipSpan?.addEvent("matrix.receive_voip_event_no_callid", {
+        "sender.userId": event.getSender(),
+      });
+      logger.error("Received call event with no call ID!");
+      return;
+    }
+
+    const call = this.callsByCallId.get(callId);
+    if (!call) {
+      this.callMembershipSpan?.addEvent(
+        "matrix.receive_voip_event_unknown_callid",
+        {
+          "sender.userId": event.getSender(),
+        }
+      );
+      logger.error("Received call event for unknown call ID " + callId);
+      return;
+    }
+
+    call.span.addEvent("matrix.receive_voip_event", {
+      "sender.userId": event.getSender(),
+      ...flattenVoipEvent(event.getContent()),
+    });
+  }
+
   public onToggleMicrophoneMuted(newValue: boolean) {
     this.callMembershipSpan?.addEvent("matrix.toggleMicMuted", {
       "matrix.microphone.muted": newValue,
@@ -197,6 +297,27 @@ export class OTelGroupCallMembership {
     });
   }
 
+  public onCallError(error: CallError, call: MatrixCall) {
+    const callTrackingInfo = this.callsByCallId.get(call.callId);
+    if (!callTrackingInfo) {
+      logger.error(`Got error for unknown call ID ${call.callId}`);
+      return;
+    }
+
+    callTrackingInfo.span.recordException(error);
+  }
+
+  public onGroupCallError(error: GroupCallError) {
+    this.callMembershipSpan?.recordException(error);
+  }
+
+  public onUndecryptableToDevice(event: MatrixEvent) {
+    this.callMembershipSpan?.addEvent("matrix.toDevice.undecryptable", {
+      "sender.userId": event.getSender(),
+    });
+  }
+
+
   public onConnectionStatsReport(
       statsReport: GroupCallStatsReport<ConnectionStatsReport>
   ) {

src/otel/otel.ts

@@ -30,7 +30,7 @@ import { Anonymity } from "../analytics/PosthogAnalytics";
 import { Config } from "../config/Config";
 import { getSetting, settingsBus } from "../settings/useSetting";
 
-const SERVICE_NAME_BASE = "element-call";
+const SERVICE_NAME = "element-call";
 
 let sharedInstance: ElementCallOpenTelemetry;
 
@@ -58,7 +58,7 @@ export class ElementCallOpenTelemetry {
     // This is how we can make Jaeger show a reaonsable service in the dropdown on the left.
     const providerConfig = {
       resource: new Resource({
-        [SemanticResourceAttributes.SERVICE_NAME]: `${SERVICE_NAME_BASE}-unauthenticated`,
+        [SemanticResourceAttributes.SERVICE_NAME]: SERVICE_NAME,
       }),
     };
     this._provider = new WebTracerProvider(providerConfig);
@@ -88,12 +88,16 @@ export class ElementCallOpenTelemetry {
 }
 
 function recheckOTelEnabledStatus(optInAnalayticsEnabled: boolean): void {
-  if (optInAnalayticsEnabled && !sharedInstance) {
+  const shouldEnable =
+    optInAnalayticsEnabled &&
+    Boolean(Config.get().opentelemetry?.collector_url);
+
+  if (shouldEnable && !sharedInstance) {
     logger.info("Starting OpenTelemetry debug reporting");
     sharedInstance = new ElementCallOpenTelemetry(
       Config.get().opentelemetry?.collector_url
     );
-  } else if (!optInAnalayticsEnabled && sharedInstance) {
+  } else if (!shouldEnable && sharedInstance) {
     logger.info("Stopping OpenTelemetry debug reporting");
     sharedInstance = undefined;
   }

src/room/GroupCallInspector.tsx

@@ -28,10 +28,20 @@ import ReactJson, { CollapsedFieldProps } from "react-json-view";
 import mermaid from "mermaid";
 import { Item } from "@react-stately/collections";
 import { MatrixEvent, IContent } from "matrix-js-sdk/src/models/event";
-import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall";
+import {
+  GroupCall,
+  GroupCallError,
+  GroupCallEvent,
+} from "matrix-js-sdk/src/webrtc/groupCall";
 import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
 import { RoomStateEvent } from "matrix-js-sdk/src/models/room-state";
-import { CallEvent, VoipEvent } from "matrix-js-sdk/src/webrtc/call";
+import {
+  CallEvent,
+  CallState,
+  CallError,
+  MatrixCall,
+  VoipEvent,
+} from "matrix-js-sdk/src/webrtc/call";
 
 import styles from "./GroupCallInspector.module.css";
 import { SelectInput } from "../input/SelectInput";
@@ -388,26 +398,49 @@ function useGroupCallState(
 
     function onReceivedVoipEvent(event: MatrixEvent) {
       dispatch({ type: ClientEvent.ReceivedVoipEvent, event });
+
+      otelGroupCallMembership?.onReceivedVoipEvent(event);
     }
 
-    function onSendVoipEvent(event: VoipEvent) {
+    function onSendVoipEvent(event: VoipEvent, call: MatrixCall) {
       dispatch({ type: CallEvent.SendVoipEvent, rawEvent: event });
 
-      otelGroupCallMembership?.onSendEvent(event);
+      otelGroupCallMembership?.onSendEvent(call, event);
+    }
+
+    function onCallStateChange(
+      newState: CallState,
+      _: CallState,
+      call: MatrixCall
+    ) {
+      otelGroupCallMembership?.onCallStateChange(call, newState);
+    }
+
+    function onCallError(error: CallError, call: MatrixCall) {
+      otelGroupCallMembership.onCallError(error, call);
+    }
+
+    function onGroupCallError(error: GroupCallError) {
+      otelGroupCallMembership.onGroupCallError(error);
     }
 
     function onUndecryptableToDevice(event: MatrixEvent) {
       dispatch({ type: ClientEvent.ReceivedVoipEvent, event });
 
       Sentry.captureMessage("Undecryptable to-device Event");
+      // probably unnecessary if it's now captured via otel?
       PosthogAnalytics.instance.eventUndecryptableToDevice.track(
         groupCall.groupCallId
       );
+
+      otelGroupCallMembership.onUndecryptableToDevice(event);
     }
 
     client.on(RoomStateEvent.Events, onUpdateRoomState);
-    //groupCall.on("calls_changed", onCallsChanged);
     groupCall.on(CallEvent.SendVoipEvent, onSendVoipEvent);
+    groupCall.on(CallEvent.State, onCallStateChange);
+    groupCall.on(CallEvent.Error, onCallError);
+    groupCall.on(GroupCallEvent.Error, onGroupCallError);
     //client.on("state", onCallsChanged);
     //client.on("hangup", onCallHangup);
     client.on(ClientEvent.ReceivedVoipEvent, onReceivedVoipEvent);
@@ -417,8 +450,10 @@ function useGroupCallState(
 
     return () => {
       client.removeListener(RoomStateEvent.Events, onUpdateRoomState);
-      //groupCall.removeListener("calls_changed", onCallsChanged);
       groupCall.removeListener(CallEvent.SendVoipEvent, onSendVoipEvent);
+      groupCall.removeListener(CallEvent.State, onCallStateChange);
+      groupCall.removeListener(CallEvent.Error, onCallError);
+      groupCall.removeListener(GroupCallEvent.Error, onGroupCallError);
       //client.removeListener("state", onCallsChanged);
       //client.removeListener("hangup", onCallHangup);
       client.removeListener(ClientEvent.ReceivedVoipEvent, onReceivedVoipEvent);

src/room/useGroupCall.ts

@@ -179,6 +179,8 @@ export function useGroupCall(
   });
 
   if (groupCallOTelMembershipGroupCallId !== groupCall.groupCallId) {
+    if (groupCallOTelMembership) groupCallOTelMembership.dispose();
+
     // If the user disables analytics, this will stay around until they leave the call
     // so analytics will be disabled once they leave.
     if (ElementCallOpenTelemetry.instance) {
@@ -206,6 +208,11 @@ export function useGroupCall(
     []
   );
 
+  const leaveCall = useCallback(() => {
+    groupCallOTelMembership?.onLeaveCall();
+    groupCall.leave();
+  }, [groupCall]);
+
   useEffect(() => {
     // disable the media action keys, otherwise audio elements get paused when
     // the user presses media keys or unplugs headphones, etc.
@@ -220,7 +227,7 @@ export function useGroupCall(
     ];
 
     for (const mediaAction of mediaActions) {
-      navigator.mediaSession.setActionHandler(
+      navigator.mediaSession?.setActionHandler(
         mediaAction,
         doNothingMediaActionCallback
       );
@@ -228,7 +235,7 @@ export function useGroupCall(
 
     return () => {
       for (const mediaAction of mediaActions) {
-        navigator.mediaSession.setActionHandler(mediaAction, null);
+        navigator.mediaSession?.setActionHandler(mediaAction, null);
       }
     };
   }, [doNothingMediaActionCallback]);
@@ -428,12 +435,12 @@ export function useGroupCall(
           GroupCallStatsReportEvent.ByteSentStats,
           onByteSentStatsReport
       );
-      groupCall.leave();
+      leaveCall();
     };
-  }, [groupCall, updateState]);
+  }, [groupCall, updateState, leaveCall]);
 
   usePageUnload(() => {
-    groupCall.leave();
+    leaveCall();
   });
 
   const initLocalCallFeed = useCallback(
@@ -452,19 +459,16 @@ export function useGroupCall(
     PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
     PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId);
 
+    // This must be called before we start trying to join the call, as we need to
+    // have started tracking by the time calls start getting created.
+    groupCallOTelMembership?.onJoinCall();
+
     groupCall.enter().catch((error) => {
       console.error(error);
       updateState({ error });
     });
-
-    groupCallOTelMembership?.onJoinCall();
   }, [groupCall, updateState]);
 
-  const leave = useCallback(() => {
-    groupCallOTelMembership?.onLeaveCall();
-    groupCall.leave();
-  }, [groupCall]);
-
   const toggleLocalVideoMuted = useCallback(() => {
     const toggleToMute = !groupCall.isLocalVideoMuted();
     groupCall.setLocalVideoMuted(toggleToMute);
@@ -597,7 +601,7 @@ export function useGroupCall(
     error,
     initLocalCallFeed,
     enter,
-    leave,
+    leave: leaveCall,
     toggleLocalVideoMuted,
     toggleMicrophoneMuted,
     toggleScreensharing,

yarn.lock

@@ -1821,7 +1821,7 @@
   resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.3.1.tgz#b50a781709c81e10701004214340f25475a171a0"
   integrity sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw==
 
-"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.3":
+"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.5":
   version "0.1.0-alpha.5"
   resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.5.tgz#60ede2c43b9d808ba8cf46085a3b347b290d9658"
   integrity sha512-2KjAgWNGfuGLNjJwsrs6gGX157vmcTfNrA4u249utgnMPbJl7QwuUqh1bGxQ0PpK06yvZjgPlkna0lTbuwtuQw==
@@ -10545,27 +10545,35 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#23837266fca5ee799b51a722f7b8eefb2f5ac140":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#042f2ed76c501c10dde98a31732fd92d862e2187":
-  version "23.5.0"
+  version "24.0.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/23837266fca5ee799b51a722f7b8eefb2f5ac140"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/042f2ed76c501c10dde98a31732fd92d862e2187"
   dependencies:
     "@babel/runtime" "^7.12.5"
-    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.3"
+    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.5"
     another-json "^0.2.0"
     bs58 "^5.0.0"
     content-type "^1.0.4"
     loglevel "^1.7.1"
     matrix-events-sdk "0.0.1"
-    matrix-widget-api "^1.0.0"
+    matrix-widget-api "^1.3.1"
     p-retry "4"
     sdp-transform "^2.14.1"
     unhomoglyph "^1.0.6"
     uuid "9"
 
-matrix-widget-api@^1.0.0:
+matrix-widget-api@^1.3.1:
-  version "1.1.1"
+  version "1.3.1"
-  resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.1.1.tgz#d3fec45033d0cbc14387a38ba92dac4dbb1be962"
+  resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.3.1.tgz#e38f404c76bb15c113909505c1c1a5b4d781c2f5"
-  integrity sha512-gNSgmgSwvOsOcWK9k2+tOhEMYBiIMwX95vMZu0JqY7apkM02xrOzUBuPRProzN8CnbIALH7e3GAhatF6QCNvtA==
+  integrity sha512-+rN6vGvnXm+fn0uq9r2KWSL/aPtehD6ObC50jYmUcEfgo8CUpf9eUurmjbRlwZkWq3XHXFuKQBUCI9UzqWg37Q==
+  dependencies:
+    "@types/events" "^3.0.0"
+    events "^3.2.0"
+
+matrix-widget-api@^1.3.1:
+  version "1.3.1"
+  resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.3.1.tgz#e38f404c76bb15c113909505c1c1a5b4d781c2f5"
+  integrity sha512-+rN6vGvnXm+fn0uq9r2KWSL/aPtehD6ObC50jYmUcEfgo8CUpf9eUurmjbRlwZkWq3XHXFuKQBUCI9UzqWg37Q==
   dependencies:
     "@types/events" "^3.0.0"
     events "^3.2.0"