Merge pull request #652 from vector-im/dbkr/fixec_screenshare

Fix screen sharing

package.json

@@ -44,7 +44,7 @@
     "i18next": "^21.10.0",
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#5a0787349d4951012eabe72f3363c17bdcda0d56",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1666419beaf08cafefdc34cced417cb0ba120b2e",
     "matrix-widget-api": "^1.0.0",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",

src/room/useGroupCall.ts

@@ -27,10 +27,11 @@ import { MatrixCall } from "matrix-js-sdk/src/webrtc/call";
 import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { useTranslation } from "react-i18next";
+import { IWidgetApiRequest } from "matrix-widget-api";
 
 import { usePageUnload } from "./usePageUnload";
 import { TranslatedError, translatedError } from "../TranslatedError";
-import { ElementWidgetActions, widget } from "../widget";
+import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget";
 
 export interface UseGroupCallReturnType {
   state: GroupCallState;
@@ -302,36 +303,84 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType {
     groupCall.setMicrophoneMuted(!groupCall.isMicrophoneMuted());
   }, [groupCall]);
 
-  const toggleScreensharing = useCallback(() => {
-    updateState({ requestingScreenshare: true });
+  const toggleScreensharing = useCallback(async () => {
+    if (!groupCall.isScreensharing()) {
+      // toggling on
+      updateState({ requestingScreenshare: true });
 
-    if (groupCall.isScreensharing()) {
-      groupCall.setScreensharingEnabled(false).then(() => {
+      try {
+        await groupCall.setScreensharingEnabled(true, {
+          audio: true,
+          throwOnFail: true,
+        });
         updateState({ requestingScreenshare: false });
-      });
-    } else {
-      widget.api.transport
-        .send(ElementWidgetActions.Screenshare, {})
-        .then(
-          (reply: { desktopCapturerSourceId: string; failed?: boolean }) => {
-            if (reply.failed) {
-              updateState({ requestingScreenshare: false });
-              return;
-            }
-
-            groupCall
-              .setScreensharingEnabled(true, {
-                audio: !reply.desktopCapturerSourceId,
-                desktopCapturerSourceId: reply.desktopCapturerSourceId,
-              })
-              .then(() => {
-                updateState({ requestingScreenshare: false });
-              });
+      } catch (e) {
+        // this will fail in Electron because getDisplayMedia just throws a permission
+        // error, so if we have a widget API, try requesting via that.
+        if (widget) {
+          const reply = await widget.api.transport.send(
+            ElementWidgetActions.ScreenshareRequest,
+            {}
+          );
+          if (!reply.pending) {
+            updateState({ requestingScreenshare: false });
           }
-        );
+        }
+      }
+    } else {
+      // toggling off
+      groupCall.setScreensharingEnabled(false);
     }
   }, [groupCall]);
 
+  const onScreenshareStart = useCallback(
+    async (ev: CustomEvent<IWidgetApiRequest>) => {
+      updateState({ requestingScreenshare: false });
+
+      const data = ev.detail.data as unknown as ScreenshareStartData;
+
+      await groupCall.setScreensharingEnabled(true, {
+        desktopCapturerSourceId: data.desktopCapturerSourceId as string,
+        audio: !data.desktopCapturerSourceId,
+      });
+      await widget.api.transport.reply(ev.detail, {});
+    },
+    [groupCall]
+  );
+
+  const onScreenshareStop = useCallback(
+    async (ev: CustomEvent<IWidgetApiRequest>) => {
+      updateState({ requestingScreenshare: false });
+      await groupCall.setScreensharingEnabled(false);
+      await widget.api.transport.reply(ev.detail, {});
+    },
+    [groupCall]
+  );
+
+  useEffect(() => {
+    if (widget) {
+      widget.lazyActions.on(
+        ElementWidgetActions.ScreenshareStart,
+        onScreenshareStart
+      );
+      widget.lazyActions.on(
+        ElementWidgetActions.ScreenshareStop,
+        onScreenshareStop
+      );
+
+      return () => {
+        widget.lazyActions.off(
+          ElementWidgetActions.ScreenshareStart,
+          onScreenshareStart
+        );
+        widget.lazyActions.off(
+          ElementWidgetActions.ScreenshareStop,
+          onScreenshareStop
+        );
+      };
+    }
+  }, [onScreenshareStart, onScreenshareStop]);
+
   const { t } = useTranslation();
 
   useEffect(() => {

src/widget.ts

@@ -30,7 +30,21 @@ export enum ElementWidgetActions {
   HangupCall = "im.vector.hangup",
   TileLayout = "io.element.tile_layout",
   SpotlightLayout = "io.element.spotlight_layout",
-  Screenshare = "io.element.screenshare",
+
+  // Element Call -> host requesting to start a screenshare
+  // (ie. expects a ScreenshareStart once the user has picked a source)
+  // Element Call -> host requesting to start a screenshare
+  // (ie. expects a ScreenshareStart once the user has picked a source)
+  // replies with { pending } where pending is true if the host has asked
+  // the user to choose a window and false if not (ie. if the host isn't
+  // running within Electron)
+  ScreenshareRequest = "io.element.screenshare_request",
+  // host -> Element Call telling EC to start screen sharing with
+  // the given source
+  ScreenshareStart = "io.element.screenshare_start",
+  // host -> Element Call telling EC to stop screen sharing, or that
+  // the user cancelled when selecting a source after a ScreenshareRequest
+  ScreenshareStop = "io.element.screenshare_stop",
 }
 
 export interface JoinCallData {
@@ -38,6 +52,10 @@ export interface JoinCallData {
   videoInput: string | null;
 }
 
+export interface ScreenshareStartData {
+  desktopCapturerSourceId: string;
+}
+
 interface WidgetHelpers {
   api: WidgetApi;
   lazyActions: LazyEventEmitter;
@@ -69,6 +87,8 @@ export const widget: WidgetHelpers | null = (() => {
         ElementWidgetActions.HangupCall,
         ElementWidgetActions.TileLayout,
         ElementWidgetActions.SpotlightLayout,
+        ElementWidgetActions.ScreenshareStart,
+        ElementWidgetActions.ScreenshareStop,
       ].forEach((action) => {
         api.on(`action:${action}`, (ev: CustomEvent<IWidgetApiRequest>) => {
           ev.preventDefault();

yarn.lock

@@ -8728,9 +8728,9 @@ matrix-events-sdk@^0.0.1-beta.7:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1-beta.7.tgz#5ffe45eba1f67cc8d7c2377736c728b322524934"
   integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#5a0787349d4951012eabe72f3363c17bdcda0d56":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#1666419beaf08cafefdc34cced417cb0ba120b2e":
   version "20.1.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/5a0787349d4951012eabe72f3363c17bdcda0d56"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/1666419beaf08cafefdc34cced417cb0ba120b2e"
   dependencies:
     "@babel/runtime" "^7.12.5"
     "@types/sdp-transform" "^2.4.5"