Implement somewhat working drag & drop and improve render memoization

src/useMergedRefs.ts

@@ -0,0 +1,16 @@
+import { MutableRefObject, RefCallback, useCallback } from "react";
+
+export const useMergedRefs = <T>(
+  ...refs: (MutableRefObject<T | null> | RefCallback<T | null>)[]
+): RefCallback<T | null> =>
+  useCallback(
+    (value) =>
+      refs.forEach((ref) => {
+        if (typeof ref === "function") {
+          ref(value);
+        } else {
+          ref.current = value;
+        }
+      }),
+    refs
+  );

src/video-grid/NewVideoGrid.tsx

@@ -1,5 +1,5 @@
 import { SpringRef, TransitionFn, useTransition } from "@react-spring/web";
-import { useDrag, useScroll } from "@use-gesture/react";
+import { EventTypes, Handler, useScroll } from "@use-gesture/react";
 import React, {
   FC,
   ReactNode,
@@ -15,6 +15,7 @@ import { VideoGridProps as Props } from "./VideoGrid";
 import { useReactiveState } from "../useReactiveState";
 import TinyQueue from "tinyqueue";
 import { zipWith } from "lodash";
+import { useMergedRefs } from "../useMergedRefs";
 
 interface Cell {
   /**
@@ -66,8 +67,10 @@ interface TileSpring {
 
 interface DragState {
   tileId: string;
-  x: number;
+  tileX: number;
-  y: number;
+  tileY: number;
+  cursorX: number;
+  cursorY: number;
 }
 
 const dijkstra = (g: Grid): number[] => {
@@ -377,7 +380,10 @@ export const NewVideoGrid: FC<Props> = ({
 }) => {
   const [slotGrid, setSlotGrid] = useState<HTMLDivElement | null>(null);
   const [slotGridGeneration, setSlotGridGeneration] = useState(0);
-  const [gridRef, gridBounds] = useMeasure();
+
+  const [gridRef1, gridBounds] = useMeasure();
+  const gridRef2 = useRef<HTMLDivElement | null>(null);
+  const gridRef = useMergedRefs(gridRef1, gridRef2);
 
   useEffect(() => {
     if (slotGrid !== null) {
@@ -549,23 +555,21 @@ export const NewVideoGrid: FC<Props> = ({
     };
   }, [grid]);
 
-  const animateDraggedTile = (endOfGesture: boolean) =>
+  const animateDraggedTile = (endOfGesture: boolean) => {
-    springRef.start((_i, controller) => {
+    const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!;
-      const { tileId, x, y } = dragState.current!;
+    const tile = tiles.find((t) => t.item.id === tileId)!;
 
-      // react-spring appears to not update a controller's item as long as the
+    springRef.start((_i, controller) => {
-      // key remains stable, so we can use it to look up the tile's ID but not
-      // its position
       if ((controller.item as Tile).item.id === tileId) {
         if (endOfGesture) {
-          const tile = tiles.find((t) => t.item.id === tileId)!;
-
           return {
             scale: 1,
             zIndex: 1,
             shadow: 1,
             x: tile.x,
             y: tile.y,
+            width: tile.width,
+            height: tile.height,
             immediate: disableAnimations || ((key) => key === "zIndex"),
             // Allow the tile's position to settle before pushing its
             // z-index back down
@@ -576,8 +580,8 @@ export const NewVideoGrid: FC<Props> = ({
             scale: 1.1,
             zIndex: 2,
             shadow: 15,
-            x,
+            x: tileX,
-            y,
+            y: tileY,
             immediate:
               disableAnimations ||
               ((key) => key === "zIndex" || key === "x" || key === "y"),
@@ -588,41 +592,78 @@ export const NewVideoGrid: FC<Props> = ({
       }
     });
 
-  const bindTile = useDrag(
+    const overTile = tiles.find(
-    ({ tap, args, delta: [dx, dy], last }) => {
+      (t) =>
-      const tileId = args[0] as string;
+        cursorX >= t.x &&
+        cursorX < t.x + t.width &&
+        cursorY >= t.y &&
+        cursorY < t.y + t.height
+    );
+    if (overTile !== undefined && overTile.item.id !== tileId) {
+      setGrid((g) => ({
+        ...g,
+        cells: g.cells.map((c) => {
+          if (c?.item === overTile.item) return { ...c, item: tile.item };
+          if (c?.item === tile.item) return { ...c, item: overTile.item };
+          return c;
+        }),
+      }));
+    }
+  };
 
-      if (tap) {
+  const onTileDrag = (
-        setGrid((g) => cycleTileSize(tileId, g));
+    tileId: string,
-      } else {
+    {
-        const tileSpring = springRef.current
+      tap,
-          .find((c) => (c.item as Tile).item.id === tileId)!
+      initial: [initialX, initialY],
-          .get();
+      delta: [dx, dy],
+      last,
+    }: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
+  ) => {
+    if (tap) {
+      setGrid((g) => cycleTileSize(tileId, g));
+    } else {
+      const tileSpring = springRef.current
+        .find((c) => (c.item as Tile).item.id === tileId)!
+        .get();
 
-        if (dragState.current === null) {
+      if (dragState.current === null) {
-          dragState.current = { tileId, x: tileSpring.x, y: tileSpring.y };
+        dragState.current = {
-        }
+          tileId,
-        dragState.current.x += dx;
+          tileX: tileSpring.x,
-        dragState.current.y += dy;
+          tileY: tileSpring.y,
-
+          cursorX: initialX - gridBounds.x,
-        animateDraggedTile(last);
+          cursorY: initialY - gridBounds.y + scrollOffset.current,
-
+        };
-        if (last) dragState.current = null;
       }
-    },
+      dragState.current.tileX += dx;
-    { filterTaps: true, pointer: { buttons: [1] } }
+      dragState.current.tileY += dy;
-  );
+      dragState.current.cursorX += dx;
+      dragState.current.cursorY += dy;
+
+      animateDraggedTile(last);
+
+      if (last) dragState.current = null;
+    }
+  };
+
+  const onTileDragRef = useRef(onTileDrag);
+  onTileDragRef.current = onTileDrag;
 
   const scrollOffset = useRef(0);
 
-  const bindGrid = useScroll(({ xy: [, y], delta: [, dy] }) => {
+  useScroll(
-    scrollOffset.current = y;
+    ({ xy: [, y], delta: [, dy] }) => {
+      scrollOffset.current = y;
 
-    if (dragState.current !== null) {
+      if (dragState.current !== null) {
-      dragState.current.y += dy;
+        dragState.current.tileY += dy;
-      animateDraggedTile(false);
+        dragState.current.cursorY += dy;
-    }
+        animateDraggedTile(false);
-  });
+      }
+    },
+    { target: gridRef2 }
+  );
 
   const slots = useMemo(() => {
     const slots = new Array<ReactNode>(items.length);
@@ -639,7 +680,7 @@ export const NewVideoGrid: FC<Props> = ({
   }
 
   return (
-    <div {...bindGrid()} ref={gridRef} className={styles.grid}>
+    <div ref={gridRef} className={styles.grid}>
       <div
         style={slotGridStyle}
         ref={setSlotGrid}
@@ -648,21 +689,14 @@ export const NewVideoGrid: FC<Props> = ({
       >
         {slots}
       </div>
-      {tileTransitions(({ shadow, width, height, ...style }, tile) =>
+      {tileTransitions((style, tile) =>
         children({
-          ...bindTile(tile.item.id),
+          ...style,
           key: tile.item.id,
-          style: {
+          targetWidth: tile.width,
-            boxShadow: shadow.to(
+          targetHeight: tile.height,
-              (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px`
-            ),
-            "--tileWidth": width.to((w) => `${w}px`),
-            "--tileHeight": height.to((h) => `${h}px`),
-            ...style,
-          },
-          width: tile.width,
-          height: tile.height,
           item: tile.item,
+          onDragRef: onTileDragRef,
         })
       )}
     </div>

src/video-grid/VideoGrid.tsx

@@ -694,14 +694,17 @@ interface DragTileData {
 
 interface ChildrenProperties extends ReactDOMAttributes {
   key: Key;
-  style: {
+  targetWidth: number;
-    scale: SpringValue<number>;
+  targetHeight: number;
-    opacity: SpringValue<number>;
-    boxShadow: Interpolation<number, string>;
-  };
-  width: number;
-  height: number;
   item: TileDescriptor;
+  opacity: SpringValue<number>;
+  scale: SpringValue<number>;
+  shadow: SpringValue<number>;
+  zIndex: SpringValue<number>;
+  x: SpringValue<number>;
+  y: SpringValue<number>;
+  width: SpringValue<number>;
+  height: SpringValue<number>;
   [index: string]: unknown;
 }
 

src/video-grid/VideoTile.tsx

@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { forwardRef } from "react";
+import React, { ForwardedRef, forwardRef } from "react";
-import { animated } from "@react-spring/web";
+import { animated, SpringValue } from "@react-spring/web";
 import classNames from "classnames";
 import { useTranslation } from "react-i18next";
 
@@ -44,9 +44,17 @@ interface Props {
   showOptions?: boolean;
   isLocal?: boolean;
   disableSpeakingIndicator?: boolean;
+  opacity: SpringValue<number>;
+  scale: SpringValue<number>;
+  shadow: SpringValue<number>;
+  zIndex: SpringValue<number>;
+  x: SpringValue<number>;
+  y: SpringValue<number>;
+  width: SpringValue<number>;
+  height: SpringValue<number>;
 }
 
-export const VideoTile = forwardRef<HTMLDivElement, Props>(
+export const VideoTile = forwardRef<HTMLElement, Props>(
   (
     {
       name,
@@ -68,6 +76,14 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       isLocal,
       // TODO: disableSpeakingIndicator is not used atm.
       disableSpeakingIndicator,
+      opacity,
+      scale,
+      shadow,
+      zIndex,
+      x,
+      y,
+      width,
+      height,
       ...rest
     },
     ref
@@ -122,7 +138,19 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
           [styles.screenshare]: screenshare,
           [styles.maximised]: maximised,
         })}
-        ref={ref}
+        style={{
+          opacity,
+          scale,
+          boxShadow: shadow.to(
+            (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px`
+          ),
+          zIndex,
+          x,
+          y,
+          "--tileWidth": width.to((w) => `${w}px`),
+          "--tileHeight": height.to((h) => `${h}px`),
+        }}
+        ref={ref as ForwardedRef<HTMLDivElement>}
         {...rest}
       >
         {toolbarButtons.length > 0 && !maximised && (

src/video-grid/VideoTileContainer.tsx

@@ -15,7 +15,7 @@ limitations under the License.
 */
 
 import { SDPStreamMetadataPurpose } from "matrix-js-sdk/src/webrtc/callEventTypes";
-import React from "react";
+import React, { FC, memo, RefObject } from "react";
 import { useCallback } from "react";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 
@@ -26,11 +26,13 @@ import { VideoTile } from "./VideoTile";
 import { VideoTileSettingsModal } from "./VideoTileSettingsModal";
 import { useModalTriggerState } from "../Modal";
 import { TileDescriptor } from "./TileDescriptor";
+import { SpringValue } from "@react-spring/web";
+import { EventTypes, Handler, useDrag } from "@use-gesture/react";
 
 interface Props {
   item: TileDescriptor;
-  width?: number;
+  targetWidth: number;
-  height?: number;
+  targetHeight: number;
   getAvatar: (
     roomMember: RoomMember,
     width: number,
@@ -42,86 +44,113 @@ interface Props {
   maximised: boolean;
   fullscreen: boolean;
   onFullscreen: (item: TileDescriptor) => void;
+  opacity: SpringValue<number>;
+  scale: SpringValue<number>;
+  shadow: SpringValue<number>;
+  zIndex: SpringValue<number>;
+  x: SpringValue<number>;
+  y: SpringValue<number>;
+  width: SpringValue<number>;
+  height: SpringValue<number>;
+  onDragRef: RefObject<
+    (
+      tileId: string,
+      state: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
+    ) => void
+  >;
 }
 
-export function VideoTileContainer({
+export const VideoTileContainer: FC<Props> = memo(
-  item,
+  ({
-  width,
+    item,
-  height,
+    targetWidth,
-  getAvatar,
+    targetHeight,
-  audioContext,
+    getAvatar,
-  audioDestination,
-  disableSpeakingIndicator,
-  maximised,
-  fullscreen,
-  onFullscreen,
-  ...rest
-}: Props) {
-  const {
-    isLocal,
-    audioMuted,
-    videoMuted,
-    localVolume,
-    hasAudio,
-    speaking,
-    stream,
-    purpose,
-  } = useCallFeed(item.callFeed);
-  const { rawDisplayName } = useRoomMemberName(item.member);
-  const [tileRef, mediaRef] = useSpatialMediaStream(
-    stream ?? null,
     audioContext,
     audioDestination,
-    localVolume,
+    disableSpeakingIndicator,
-    // The feed is muted if it's local audio (because we don't want our own audio,
+    maximised,
-    // but it's a hook and we can't call it conditionally so we're stuck with it)
+    fullscreen,
-    // or if there's a maximised feed in which case we always render audio via audio
+    onFullscreen,
-    // elements because we wire it up at the video tile container level and only one
+    onDragRef,
-    // video tile container is displayed.
+    ...rest
-    isLocal || maximised
+  }) => {
-  );
+    const {
-  const {
+      isLocal,
-    modalState: videoTileSettingsModalState,
+      audioMuted,
-    modalProps: videoTileSettingsModalProps,
+      videoMuted,
-  } = useModalTriggerState();
+      localVolume,
-  const onOptionsPress = () => {
+      hasAudio,
-    videoTileSettingsModalState.open();
+      speaking,
-  };
+      stream,
+      purpose,
+    } = useCallFeed(item.callFeed);
+    const { rawDisplayName } = useRoomMemberName(item.member);
 
-  const onFullscreenCallback = useCallback(() => {
+    const [tileRef, mediaRef] = useSpatialMediaStream(
-    onFullscreen(item);
+      stream ?? null,
-  }, [onFullscreen, item]);
+      audioContext,
+      audioDestination,
+      localVolume,
+      // The feed is muted if it's local audio (because we don't want our own audio,
+      // but it's a hook and we can't call it conditionally so we're stuck with it)
+      // or if there's a maximised feed in which case we always render audio via audio
+      // elements because we wire it up at the video tile container level and only one
+      // video tile container is displayed.
+      isLocal || maximised
+    );
 
-  // Firefox doesn't respect the disablePictureInPicture attribute
+    useDrag((state) => onDragRef.current!(item.id, state), {
-  // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
+      target: tileRef,
+      filterTaps: true,
+      pointer: { buttons: [1] },
+    });
 
-  return (
+    const {
-    <>
+      modalState: videoTileSettingsModalState,
-      <VideoTile
+      modalProps: videoTileSettingsModalProps,
-        isLocal={isLocal}
+    } = useModalTriggerState();
-        speaking={speaking && !disableSpeakingIndicator}
+    const onOptionsPress = () => {
-        audioMuted={audioMuted}
+      videoTileSettingsModalState.open();
-        videoMuted={videoMuted}
+    };
-        screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
+
-        name={rawDisplayName}
+    const onFullscreenCallback = useCallback(() => {
-        connectionState={item.connectionState}
+      onFullscreen(item);
-        ref={tileRef}
+    }, [onFullscreen, item]);
-        mediaRef={mediaRef}
+
-        avatar={getAvatar && getAvatar(item.member, width, height)}
+    // Firefox doesn't respect the disablePictureInPicture attribute
-        onOptionsPress={onOptionsPress}
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
-        localVolume={localVolume}
+
-        hasAudio={hasAudio}
+    return (
-        maximised={maximised}
+      <>
-        fullscreen={fullscreen}
+        <VideoTile
-        onFullscreen={onFullscreenCallback}
+          isLocal={isLocal}
-        {...rest}
+          speaking={speaking && !disableSpeakingIndicator}
-      />
+          audioMuted={audioMuted}
-      {videoTileSettingsModalState.isOpen && !maximised && item.callFeed && (
+          videoMuted={videoMuted}
-        <VideoTileSettingsModal
+          screenshare={purpose === SDPStreamMetadataPurpose.Screenshare}
-          {...videoTileSettingsModalProps}
+          name={rawDisplayName}
-          feed={item.callFeed}
+          connectionState={item.connectionState}
+          ref={tileRef}
+          mediaRef={mediaRef}
+          avatar={
+            getAvatar && getAvatar(item.member, targetWidth, targetHeight)
+          }
+          onOptionsPress={onOptionsPress}
+          localVolume={localVolume}
+          hasAudio={hasAudio}
+          maximised={maximised}
+          fullscreen={fullscreen}
+          onFullscreen={onFullscreenCallback}
+          {...rest}
         />
-      )}
+        {videoTileSettingsModalState.isOpen && !maximised && item.callFeed && (
-    </>
+          <VideoTileSettingsModal
-  );
+            {...videoTileSettingsModalProps}
-}
+            feed={item.callFeed}
+          />
+        )}
+      </>
+    );
+  }
+);

src/video-grid/useMediaStream.ts

@@ -158,8 +158,8 @@ export const useSpatialMediaStream = (
   audioDestination: AudioNode,
   localVolume: number,
   mute = false
-): [RefObject<HTMLDivElement>, RefObject<MediaElement>] => {
+): [RefObject<HTMLElement>, RefObject<MediaElement>] => {
-  const tileRef = useRef<HTMLDivElement | null>(null);
+  const tileRef = useRef<HTMLElement | null>(null);
   const [spatialAudio] = useSpatialAudio();
 
   // This media stream is only used for the video - the audio goes via the audio