Merge branch 'livekit-experiment' into livekit-load-test

src/video-grid/NewVideoGrid.module.css

@@ -0,0 +1,48 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.grid {
+  contain: strict;
+  position: relative;
+  flex-grow: 1;
+  padding: 0 20px;
+  overflow-y: auto;
+  overflow-x: hidden;
+}
+
+.slotGrid {
+  position: relative;
+  display: grid;
+  grid-auto-rows: 163px;
+  gap: 8px;
+  padding-bottom: var(--footerHeight);
+}
+
+.slot {
+  contain: strict;
+}
+
+@media (min-width: 800px) {
+  .grid {
+    padding: 0 22px;
+  }
+
+  .slotGrid {
+    grid-auto-rows: 183px;
+    column-gap: 18px;
+    row-gap: 21px;
+  }
+}

src/video-grid/NewVideoGrid.tsx

@@ -0,0 +1,470 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { SpringRef, TransitionFn, useTransition } from "@react-spring/web";
+import { EventTypes, Handler, useScroll } from "@use-gesture/react";
+import React, {
+  Dispatch,
+  FC,
+  ReactNode,
+  SetStateAction,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import useMeasure from "react-use-measure";
+import { zipWith } from "lodash";
+
+import styles from "./NewVideoGrid.module.css";
+import {
+  VideoGridProps as Props,
+  TileSpring,
+  TileDescriptor,
+} from "./VideoGrid";
+import { useReactiveState } from "../useReactiveState";
+import { useMergedRefs } from "../useMergedRefs";
+import {
+  Grid,
+  Cell,
+  row,
+  column,
+  fillGaps,
+  forEachCellInArea,
+  cycleTileSize,
+  appendItems,
+} from "./model";
+
+interface GridState extends Grid {
+  /**
+   * The ID of the current state of the grid.
+   */
+  generation: number;
+}
+
+const useGridState = (
+  columns: number | null,
+  items: TileDescriptor<unknown>[]
+): [GridState | null, Dispatch<SetStateAction<Grid>>] => {
+  const [grid, setGrid_] = useReactiveState<GridState | null>(
+    (prevGrid = null) => {
+      if (prevGrid === null) {
+        // We can't do anything if the column count isn't known yet
+        if (columns === null) {
+          return null;
+        } else {
+          prevGrid = { generation: 0, columns, cells: [] };
+        }
+      }
+
+      // Step 1: Update tiles that still exist, and remove tiles that have left
+      // the grid
+      const itemsById = new Map(items.map((i) => [i.id, i]));
+      const grid1: Grid = {
+        ...prevGrid,
+        cells: prevGrid.cells.map((c) => {
+          if (c === undefined) return undefined;
+          const item = itemsById.get(c.item.id);
+          return item === undefined ? undefined : { ...c, item };
+        }),
+      };
+
+      // Step 2: Backfill gaps left behind by removed tiles
+      const grid2 = fillGaps(grid1);
+
+      // Step 3: Add new tiles to the end of the grid
+      const existingItemIds = new Set(
+        grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id)
+      );
+      const newItems = items.filter((i) => !existingItemIds.has(i.id));
+      const grid3 = appendItems(newItems, grid2);
+
+      return { ...grid3, generation: prevGrid.generation + 1 };
+    },
+    [columns, items]
+  );
+
+  const setGrid: Dispatch<SetStateAction<Grid>> = useCallback(
+    (action) => {
+      if (typeof action === "function") {
+        setGrid_((prevGrid) =>
+          prevGrid === null
+            ? null
+            : {
+                ...(action as (prev: Grid) => Grid)(prevGrid),
+                generation: prevGrid.generation + 1,
+              }
+        );
+      } else {
+        setGrid_((prevGrid) => ({
+          ...action,
+          generation: prevGrid?.generation ?? 1,
+        }));
+      }
+    },
+    [setGrid_]
+  );
+
+  return [grid, setGrid];
+};
+
+interface Rect {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+interface Tile extends Rect {
+  item: TileDescriptor<unknown>;
+}
+
+interface DragState {
+  tileId: string;
+  tileX: number;
+  tileY: number;
+  cursorX: number;
+  cursorY: number;
+}
+
+/**
+ * An interactive, animated grid of video tiles.
+ */
+export const NewVideoGrid: FC<Props<unknown>> = ({
+  items,
+  disableAnimations,
+  children,
+}) => {
+  // Overview: This component lays out tiles by rendering an invisible template
+  // grid of "slots" for tiles to go in. Once rendered, it uses the DOM API to
+  // get the dimensions of each slot, feeding these numbers back into
+  // react-spring to let the actual tiles move freely atop the template.
+
+  // To know when the rendered grid becomes consistent with the layout we've
+  // requested, we give it a data-generation attribute which holds the ID of the
+  // most recently rendered generation of the grid, and watch it with a
+  // MutationObserver.
+
+  const [slotGrid, setSlotGrid] = useState<HTMLDivElement | null>(null);
+  const [slotGridGeneration, setSlotGridGeneration] = useState(0);
+
+  useEffect(() => {
+    if (slotGrid !== null) {
+      setSlotGridGeneration(
+        parseInt(slotGrid.getAttribute("data-generation")!)
+      );
+
+      const observer = new MutationObserver((mutations) => {
+        if (mutations.some((m) => m.type === "attributes")) {
+          setSlotGridGeneration(
+            parseInt(slotGrid.getAttribute("data-generation")!)
+          );
+        }
+      });
+
+      observer.observe(slotGrid, { attributes: true });
+      return () => observer.disconnect();
+    }
+  }, [slotGrid, setSlotGridGeneration]);
+
+  const [gridRef1, gridBounds] = useMeasure();
+  const gridRef2 = useRef<HTMLDivElement | null>(null);
+  const gridRef = useMergedRefs(gridRef1, gridRef2);
+
+  const slotRects = useMemo(() => {
+    if (slotGrid === null) return [];
+
+    const slots = slotGrid.getElementsByClassName(styles.slot);
+    const rects = new Array<Rect>(slots.length);
+    for (let i = 0; i < slots.length; i++) {
+      const slot = slots[i] as HTMLElement;
+      rects[i] = {
+        x: slot.offsetLeft,
+        y: slot.offsetTop,
+        width: slot.offsetWidth,
+        height: slot.offsetHeight,
+      };
+    }
+
+    return rects;
+    // The rects may change due to the grid being resized or rerendered, but
+    // eslint can't statically verify this
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [slotGrid, slotGridGeneration, gridBounds]);
+
+  const [columns] = useReactiveState<number | null>(
+    // Since grid resizing isn't implemented yet, pick a column count on mount
+    // and stick to it
+    (prevColumns) =>
+      prevColumns !== undefined && prevColumns !== null
+        ? prevColumns
+        : // The grid bounds might not be known yet
+        gridBounds.width === 0
+        ? null
+        : Math.max(2, Math.floor(gridBounds.width * 0.0045)),
+    [gridBounds]
+  );
+
+  const [grid, setGrid] = useGridState(columns, items);
+
+  const [tiles] = useReactiveState<Tile[]>(
+    (prevTiles) => {
+      // If React hasn't yet rendered the current generation of the grid, skip
+      // the update, because grid and slotRects will be out of sync
+      if (slotGridGeneration !== grid?.generation) return prevTiles ?? [];
+
+      const tileCells = grid.cells.filter((c) => c?.origin) as Cell[];
+      const tileRects = new Map<TileDescriptor<unknown>, Rect>(
+        zipWith(tileCells, slotRects, (cell, rect) => [cell.item, rect])
+      );
+      return items.map((item) => ({ ...tileRects.get(item)!, item }));
+    },
+    [slotRects, grid, slotGridGeneration]
+  );
+
+  // Drag state is stored in a ref rather than component state, because we use
+  // react-spring's imperative API during gestures to improve responsiveness
+  const dragState = useRef<DragState | null>(null);
+
+  const [tileTransitions, springRef] = useTransition(
+    tiles,
+    () => ({
+      key: ({ item }: Tile) => item.id,
+      from: ({ x, y, width, height }: Tile) => ({
+        opacity: 0,
+        scale: 0,
+        shadow: 1,
+        shadowSpread: 0,
+        zIndex: 1,
+        x,
+        y,
+        width,
+        height,
+        immediate: disableAnimations,
+      }),
+      enter: { opacity: 1, scale: 1, immediate: disableAnimations },
+      update: ({ item, x, y, width, height }: Tile) =>
+        item.id === dragState.current?.tileId
+          ? {}
+          : {
+              x,
+              y,
+              width,
+              height,
+              immediate: disableAnimations,
+            },
+      leave: { opacity: 0, scale: 0, immediate: disableAnimations },
+      config: { mass: 0.7, tension: 252, friction: 25 },
+    })
+    // react-spring's types are bugged and can't infer the spring type
+  ) as unknown as [TransitionFn<Tile, TileSpring>, SpringRef<TileSpring>];
+
+  // Because we're using react-spring in imperative mode, we're responsible for
+  // firing animations manually whenever the tiles array updates
+  useEffect(() => {
+    springRef.start();
+  }, [tiles, springRef]);
+
+  const animateDraggedTile = (endOfGesture: boolean) => {
+    const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!;
+    const tile = tiles.find((t) => t.item.id === tileId)!;
+
+    springRef.start((_i, controller) => {
+      if ((controller.item as Tile).item.id === tileId) {
+        if (endOfGesture) {
+          return {
+            scale: 1,
+            zIndex: 1,
+            shadow: 1,
+            x: tile.x,
+            y: tile.y,
+            width: tile.width,
+            height: tile.height,
+            immediate: disableAnimations || ((key) => key === "zIndex"),
+            // Allow the tile's position to settle before pushing its
+            // z-index back down
+            delay: (key) => (key === "zIndex" ? 500 : 0),
+          };
+        } else {
+          return {
+            scale: 1.1,
+            zIndex: 2,
+            shadow: 15,
+            x: tileX,
+            y: tileY,
+            immediate:
+              disableAnimations ||
+              ((key) => key === "zIndex" || key === "x" || key === "y"),
+          };
+        }
+      } else {
+        return {};
+      }
+    });
+
+    const overTile = tiles.find(
+      (t) =>
+        cursorX >= t.x &&
+        cursorX < t.x + t.width &&
+        cursorY >= t.y &&
+        cursorY < t.y + t.height
+    );
+    if (overTile !== undefined && overTile.item.id !== tileId) {
+      setGrid((g) => ({
+        ...g!,
+        cells: g!.cells.map((c) => {
+          if (c?.item === overTile.item) return { ...c, item: tile.item };
+          if (c?.item === tile.item) return { ...c, item: overTile.item };
+          return c;
+        }),
+      }));
+    }
+  };
+
+  // Callback for useDrag. We could call useDrag here, but the default
+  // pattern of spreading {...bind()} across the children to bind the gesture
+  // ends up breaking memoization and ruining this component's performance.
+  // Instead, we pass this callback to each tile via a ref, to let them bind the
+  // gesture using the much more sensible ref-based method.
+  const onTileDrag = (
+    tileId: string,
+    {
+      tap,
+      initial: [initialX, initialY],
+      delta: [dx, dy],
+      last,
+    }: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
+  ) => {
+    if (tap) {
+      setGrid((g) => cycleTileSize(tileId, g!));
+    } else {
+      const tileSpring = springRef.current
+        .find((c) => (c.item as Tile).item.id === tileId)!
+        .get();
+
+      if (dragState.current === null) {
+        dragState.current = {
+          tileId,
+          tileX: tileSpring.x,
+          tileY: tileSpring.y,
+          cursorX: initialX - gridBounds.x,
+          cursorY: initialY - gridBounds.y + scrollOffset.current,
+        };
+      }
+      dragState.current.tileX += dx;
+      dragState.current.tileY += dy;
+      dragState.current.cursorX += dx;
+      dragState.current.cursorY += dy;
+
+      animateDraggedTile(last);
+
+      if (last) dragState.current = null;
+    }
+  };
+
+  const onTileDragRef = useRef(onTileDrag);
+  onTileDragRef.current = onTileDrag;
+
+  const scrollOffset = useRef(0);
+
+  useScroll(
+    ({ xy: [, y], delta: [, dy] }) => {
+      scrollOffset.current = y;
+
+      if (dragState.current !== null) {
+        dragState.current.tileY += dy;
+        dragState.current.cursorY += dy;
+        animateDraggedTile(false);
+      }
+    },
+    { target: gridRef2 }
+  );
+
+  const slotGridStyle = useMemo(() => {
+    if (grid === null) return {};
+
+    const areas = new Array<(number | null)[]>(
+      Math.ceil(grid.cells.length / grid.columns)
+    );
+    for (let i = 0; i < areas.length; i++)
+      areas[i] = new Array<number | null>(grid.columns).fill(null);
+
+    let slotId = 0;
+    for (let i = 0; i < grid.cells.length; i++) {
+      const cell = grid.cells[i];
+      if (cell?.origin) {
+        const slotEnd = i + cell.columns - 1 + grid.columns * (cell.rows - 1);
+        forEachCellInArea(
+          i,
+          slotEnd,
+          grid,
+          (_c, j) => (areas[row(j, grid)][column(j, grid)] = slotId)
+        );
+        slotId++;
+      }
+    }
+
+    return {
+      gridTemplateAreas: areas
+        .map(
+          (row) =>
+            `'${row
+              .map((slotId) => (slotId === null ? "." : `s${slotId}`))
+              .join(" ")}'`
+        )
+        .join(" "),
+      gridTemplateColumns: `repeat(${columns}, 1fr)`,
+    };
+  }, [grid, columns]);
+
+  const slots = useMemo(() => {
+    const slots = new Array<ReactNode>(items.length);
+    for (let i = 0; i < items.length; i++)
+      slots[i] = (
+        <div className={styles.slot} key={i} style={{ gridArea: `s${i}` }} />
+      );
+    return slots;
+  }, [items.length]);
+
+  // Render nothing if the grid has yet to be generated
+  if (grid === null) {
+    return <div ref={gridRef} className={styles.grid} />;
+  }
+
+  return (
+    <div ref={gridRef} className={styles.grid}>
+      <div
+        style={slotGridStyle}
+        ref={setSlotGrid}
+        className={styles.slotGrid}
+        data-generation={grid.generation}
+      >
+        {slots}
+      </div>
+      {tileTransitions((style, tile) =>
+        children({
+          ...style,
+          key: tile.item.id,
+          targetWidth: tile.width,
+          targetHeight: tile.height,
+          data: tile.item.data,
+          onDragRef: onTileDragRef,
+        })
+      )}
+    </div>
+  );
+};

src/video-grid/VideoGrid.module.css

@@ -19,4 +19,5 @@ limitations under the License.
   overflow: hidden;
   flex: 1;
   touch-action: none;
+  margin-bottom: var(--footerHeight);
 }

src/video-grid/VideoGrid.tsx

@@ -14,11 +14,29 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { Key, useCallback, useEffect, useRef, useState } from "react";
-import { FullGestureState, useDrag, useGesture } from "@use-gesture/react";
-import { Interpolation, SpringValue, useSprings } from "@react-spring/web";
+import React, {
+  Key,
+  RefObject,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import {
+  EventTypes,
+  FullGestureState,
+  Handler,
+  useDrag,
+  useGesture,
+} from "@use-gesture/react";
+import {
+  SpringRef,
+  SpringValue,
+  SpringValues,
+  useSprings,
+} from "@react-spring/web";
 import useMeasure from "react-use-measure";
-import { ResizeObserver } from "@juggle/resize-observer";
+import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer";
 import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/types";
 
 import styles from "./VideoGrid.module.css";
@@ -40,6 +58,18 @@ interface Tile<T> {
   focused: boolean;
 }
 
+export interface TileSpring {
+  opacity: number;
+  scale: number;
+  shadow: number;
+  shadowSpread: number;
+  zIndex: number;
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
 type LayoutDirection = "vertical" | "horizontal";
 
 export function useVideoGridLayout(hasScreenshareFeeds: boolean): {
@@ -153,8 +183,16 @@ function getOneOnOneLayoutTilePositions(
   const gridAspectRatio = gridWidth / gridHeight;
 
   const smallPip = gridAspectRatio < 1 || gridWidth < 700;
-  const pipWidth = smallPip ? 114 : 230;
-  const pipHeight = smallPip ? 163 : 155;
+  const maxPipWidth = smallPip ? 114 : 230;
+  const maxPipHeight = smallPip ? 163 : 155;
+  // Cap the PiP size at 1/3 the remote tile size, preserving aspect ratio
+  const pipScaleFactor = Math.min(
+    1,
+    remotePosition.width / 3 / maxPipWidth,
+    remotePosition.height / 3 / maxPipHeight
+  );
+  const pipWidth = maxPipWidth * pipScaleFactor;
+  const pipHeight = maxPipHeight * pipScaleFactor;
   const pipGap = getPipGap(gridAspectRatio, gridWidth);
 
   const pipMinX = remotePosition.x + pipGap;
@@ -689,19 +727,28 @@ interface DragTileData {
   y: number;
 }
 
-interface ChildrenProperties<T> extends ReactDOMAttributes {
+export interface ChildrenProperties<T> extends ReactDOMAttributes {
   key: Key;
   data: T;
-  style: {
-    scale: SpringValue<number>;
-    opacity: SpringValue<number>;
-    boxShadow: Interpolation<number, string>;
-  };
-  width: number;
-  height: number;
+  targetWidth: number;
+  targetHeight: number;
+  opacity: SpringValue<number>;
+  scale: SpringValue<number>;
+  shadow: SpringValue<number>;
+  zIndex: SpringValue<number>;
+  x: SpringValue<number>;
+  y: SpringValue<number>;
+  width: SpringValue<number>;
+  height: SpringValue<number>;
+  onDragRef?: RefObject<
+    (
+      tileId: string,
+      state: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
+    ) => void
+  >;
 }
 
-interface VideoGridProps<T> {
+export interface VideoGridProps<T> {
   items: TileDescriptor<T>[];
   layout: Layout;
   disableAnimations: boolean;
@@ -740,7 +787,13 @@ export function VideoGrid<T>({
   const lastLayoutRef = useRef<Layout>(layout);
   const isMounted = useIsMounted();
 
-  const [gridRef, gridBounds] = useMeasure({ polyfill: ResizeObserver });
+  // The 'polyfill' argument to useMeasure is not a polyfill at all but is the impl that is always used
+  // if passed, whether the browser has native support or not, so pass in either the browser native
+  // version or the ponyfill (yes, pony) because Juggle's resizeobserver ponyfill is being weirdly
+  // buggy for me on my dev env my never updating the size until the window resizes.
+  const [gridRef, gridBounds] = useMeasure({
+    polyfill: window.ResizeObserver ?? JuggleResizeObserver,
+  });
 
   useEffect(() => {
     setTileState(({ tiles, ...rest }) => {
@@ -868,6 +921,8 @@ export function VideoGrid<T>({
       // Whether the tile positions were valid at the time of the previous
       // animation
       const tilePositionsWereValid = tilePositionsValid.current;
+      const oneOnOneLayout =
+        tiles.length === 2 && !tiles.some((t) => t.focused);
 
       return (tileIndex: number) => {
         const tile = tiles[tileIndex];
@@ -887,16 +942,19 @@ export function VideoGrid<T>({
             opacity: 1,
             zIndex: 2,
             shadow: 15,
+            shadowSpread: 0,
             immediate: (key: string) =>
               disableAnimations ||
               key === "zIndex" ||
               key === "x" ||
               key === "y" ||
-              key === "shadow",
+              key === "shadow" ||
+              key === "shadowSpread",
             from: {
               shadow: 0,
               scale: 0,
               opacity: 0,
+              zIndex: 0,
             },
             reset: false,
           };
@@ -920,6 +978,7 @@ export function VideoGrid<T>({
             shadow: number;
             scale: number;
             opacity: number;
+            zIndex?: number;
             x?: number;
             y?: number;
             width?: number;
@@ -948,10 +1007,14 @@ export function VideoGrid<T>({
             opacity: remove ? 0 : 1,
             zIndex: tilePosition.zIndex,
             shadow: 1,
+            shadowSpread: oneOnOneLayout && tile.item.local ? 1 : 0,
             from,
             reset,
             immediate: (key: string) =>
-              disableAnimations || key === "zIndex" || key === "shadow",
+              disableAnimations ||
+              key === "zIndex" ||
+              key === "shadow" ||
+              key === "shadowSpread",
             // If we just stopped dragging a tile, give it time for the
             // animation to settle before pushing its z-index back down
             delay: (key: string) => (key === "zIndex" ? 500 : 0),
@@ -966,7 +1029,8 @@ export function VideoGrid<T>({
     tilePositions,
     tiles,
     scrollPosition,
-  ]);
+    // react-spring's types are bugged and can't infer the spring type
+  ]) as unknown as [SpringValues<TileSpring>[], SpringRef<TileSpring>];
 
   const onTap = useCallback(
     (tileKey: Key) => {
@@ -1175,22 +1239,17 @@ export function VideoGrid<T>({
 
   return (
     <div className={styles.videoGrid} ref={gridRef} {...bindGrid()}>
-      {springs.map(({ shadow, ...style }, i) => {
+      {springs.map((style, i) => {
         const tile = tiles[i];
         const tilePosition = tilePositions[tile.order];
 
         return createChild({
           ...bindTile(tile.key),
+          ...style,
           key: tile.key,
           data: tile.item.data,
-          style: {
-            boxShadow: shadow.to(
-              (s) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px 0px`
-            ),
-            ...style,
-          },
-          width: tilePosition.width,
-          height: tilePosition.height,
+          targetWidth: tilePosition.width,
+          targetHeight: tilePosition.height,
         });
       })}
     </div>

src/video-grid/VideoTile.module.css

@@ -16,11 +16,16 @@ limitations under the License.
 
 .videoTile {
   position: absolute;
-  will-change: transform, width, height, opacity, box-shadow;
-  border-radius: 20px;
+  contain: strict;
+  top: 0;
+  width: var(--tileWidth);
+  height: var(--tileHeight);
+  --tileRadius: 8px;
+  border-radius: var(--tileRadius);
+  box-shadow: rgba(0, 0, 0, 0.5) 0px var(--tileShadow)
+    calc(2 * var(--tileShadow)) var(--tileShadowSpread);
   overflow: hidden;
   cursor: pointer;
-  touch-action: none;
 
   /* HACK: This has no visual effect due to the short duration, but allows the
   JS to detect movement via the transform property for audio spatialization */
@@ -28,9 +33,6 @@ limitations under the License.
 }
 
 .videoTile * {
-  touch-action: none;
-  -moz-user-select: none;
-  -webkit-user-drag: none;
   user-select: none;
 }
 
@@ -45,15 +47,21 @@ limitations under the License.
   transform: scaleX(-1);
 }
 
-.videoTile.speaking::after {
+.videoTile::after {
   position: absolute;
   top: -1px;
   left: -1px;
   right: -1px;
   bottom: -1px;
   content: "";
-  border-radius: 20px;
+  border-radius: var(--tileRadius);
   box-shadow: inset 0 0 0 4px var(--accent) !important;
+  opacity: 0;
+  transition: opacity ease 0.15s;
+}
+
+.videoTile.speaking::after {
+  opacity: 1;
 }
 
 .videoTile.maximised {
@@ -83,6 +91,12 @@ limitations under the License.
   z-index: 1;
 }
 
+.infoBubble > svg {
+  height: 16px;
+  width: 16px;
+  margin-right: 4px;
+}
+
 .toolbar {
   position: absolute;
   top: 0;
@@ -126,10 +140,6 @@ limitations under the License.
   bottom: 16px;
 }
 
-.memberName > * {
-  margin-right: 6px;
-}
-
 .memberName > :last-child {
   margin-right: 0px;
 }
@@ -143,13 +153,6 @@ limitations under the License.
   white-space: nowrap;
 }
 
-.videoMutedAvatar {
-  position: absolute;
-  left: 50%;
-  top: 50%;
-  transform: translate(-50%, -50%);
-}
-
 .videoMutedOverlay {
   width: 100%;
   height: 100%;
@@ -186,3 +189,9 @@ limitations under the License.
   left: 16px;
   background-color: rgba(0, 0, 0, 0.5);
 }
+
+@media (min-width: 800px) {
+  .videoTile {
+    --tileRadius: 20px;
+  }
+}

src/video-grid/VideoTile.tsx

@@ -14,8 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import React, { forwardRef } from "react";
-import { animated } from "@react-spring/web";
+import React, { ForwardedRef, forwardRef } from "react";
+import { animated, SpringValue } from "@react-spring/web";
 import classNames from "classnames";
 import { useTranslation } from "react-i18next";
 import { LocalParticipant, RemoteParticipant, Track } from "livekit-client";
@@ -26,8 +26,8 @@ import {
 } from "@livekit/components-react";
 
 import styles from "./VideoTile.module.css";
+import { ReactComponent as MicIcon } from "../icons/Mic.svg";
 import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
-import { ReactComponent as VideoMutedIcon } from "../icons/VideoMuted.svg";
 
 export enum TileContent {
   UserMedia = "user-media",
@@ -35,16 +35,46 @@ export enum TileContent {
 }
 
 interface Props {
-  avatar?: JSX.Element;
-  className?: string;
-
   name: string;
   sfuParticipant: LocalParticipant | RemoteParticipant;
   content: TileContent;
+
+  // TODO: Refactor this set of props.
+  // See https://github.com/vector-im/element-call/pull/1099#discussion_r1226863404
+  avatar?: JSX.Element;
+  className?: string;
+  opacity?: SpringValue<number>;
+  scale?: SpringValue<number>;
+  shadow?: SpringValue<number>;
+  shadowSpread?: SpringValue<number>;
+  zIndex?: SpringValue<number>;
+  x?: SpringValue<number>;
+  y?: SpringValue<number>;
+  width?: SpringValue<number>;
+  height?: SpringValue<number>;
 }
 
-export const VideoTile = forwardRef<HTMLDivElement, Props>(
-  ({ name, avatar, className, sfuParticipant, content, ...rest }, ref) => {
+export const VideoTile = forwardRef<HTMLElement, Props>(
+  (
+    {
+      name,
+      sfuParticipant,
+      content,
+      avatar,
+      className,
+      opacity,
+      scale,
+      shadow,
+      shadowSpread,
+      zIndex,
+      x,
+      y,
+      width,
+      height,
+      ...rest
+    },
+    ref
+  ) => {
     const { t } = useTranslation();
 
     const audioEl = React.useRef<HTMLAudioElement>(null);
@@ -66,7 +96,22 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
           [styles.muted]: microphoneMuted,
           [styles.screenshare]: false,
         })}
-        ref={ref}
+        style={{
+          opacity,
+          scale,
+          zIndex,
+          x,
+          y,
+          // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+          // @ts-ignore React does in fact support assigning custom properties,
+          // but React's types say no
+          "--tileWidth": width?.to((w) => `${w}px`),
+          "--tileHeight": height?.to((h) => `${h}px`),
+          "--tileShadow": shadow?.to((s) => `${s}px`),
+          "--tileShadowSpread": shadowSpread?.to((s) => `${s}px`),
+        }}
+        ref={ref as ForwardedRef<HTMLDivElement>}
+        data-testid="videoTile"
         {...rest}
       >
         {!sfuParticipant.isCameraEnabled && (
@@ -75,19 +120,17 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             {avatar}
           </>
         )}
-        {!false &&
-          (sfuParticipant.isScreenShareEnabled ? (
-            <div className={styles.presenterLabel}>
-              <span>{t("{{name}} is presenting", { name })}</span>
-            </div>
-          ) : (
-            <div className={classNames(styles.infoBubble, styles.memberName)}>
-              {microphoneMuted && <MicMutedIcon />}
-              {!sfuParticipant.isCameraEnabled && <VideoMutedIcon />}
-              <span title={name}>{name}</span>
-              <ConnectionQualityIndicator participant={sfuParticipant} />
-            </div>
-          ))}
+        {sfuParticipant.isScreenShareEnabled ? (
+          <div className={styles.presenterLabel}>
+            <span>{t("{{name}} is presenting", { name })}</span>
+          </div>
+        ) : (
+          <div className={classNames(styles.infoBubble, styles.memberName)}>
+            {microphoneMuted ? <MicMutedIcon /> : <MicIcon />}
+            <span title={name}>{name}</span>
+            <ConnectionQualityIndicator participant={sfuParticipant} />
+          </div>
+        )}
         <VideoTrack
           participant={sfuParticipant}
           source={

src/video-grid/VideoTileContainer.tsx

@@ -20,6 +20,8 @@ import {
   RoomMemberEvent,
 } from "matrix-js-sdk/src/models/room-member";
 import { LocalParticipant, RemoteParticipant } from "livekit-client";
+import { SpringValue } from "@react-spring/web";
+import { EventTypes, Handler, useDrag } from "@use-gesture/react";
 
 import { TileContent, VideoTile } from "./VideoTile";
 import { Avatar } from "../Avatar";
@@ -33,49 +35,81 @@ export interface ItemData {
 
 interface Props {
   item: ItemData;
-  width?: number;
-  height?: number;
+
+  // TODO: Refactor this set of props.
+  // See https://github.com/vector-im/element-call/pull/1099#discussion_r1226863404
+  id: string;
+  targetWidth: number;
+  targetHeight: number;
+  opacity?: SpringValue<number>;
+  scale?: SpringValue<number>;
+  shadow?: SpringValue<number>;
+  shadowSpread?: SpringValue<number>;
+  zIndex?: SpringValue<number>;
+  x?: SpringValue<number>;
+  y?: SpringValue<number>;
+  width?: SpringValue<number>;
+  height?: SpringValue<number>;
+  onDragRef?: React.RefObject<
+    (
+      tileId: string,
+      state: Parameters<Handler<"drag", EventTypes["drag"]>>[0]
+    ) => void
+  >;
 }
 
-export function VideoTileContainer({ item, width, height, ...rest }: Props) {
-  const [displayName, setDisplayName] = React.useState<string>("[👻]");
+export const VideoTileContainer: React.FC<Props> = React.memo(
+  ({ item, id, targetWidth, targetHeight, onDragRef, ...rest }) => {
+    // Handle display name changes.
+    const [displayName, setDisplayName] = React.useState<string>("[👻]");
+    React.useEffect(() => {
+      const member = item.member;
 
-  React.useEffect(() => {
-    const member = item.member;
-
-    if (member) {
-      setDisplayName(member.rawDisplayName);
-
-      const updateName = () => {
+      if (member) {
         setDisplayName(member.rawDisplayName);
-      };
 
-      member!.on(RoomMemberEvent.Name, updateName);
-      return () => {
-        member!.removeListener(RoomMemberEvent.Name, updateName);
-      };
-    }
-  }, [item.member]);
+        const updateName = () => {
+          setDisplayName(member.rawDisplayName);
+        };
 
-  const avatar = (
-    <Avatar
-      key={item.member?.userId}
-      size={Math.round(Math.min(width ?? 0, height ?? 0) / 2)}
-      src={item.member?.getMxcAvatarUrl()}
-      fallback={displayName.slice(0, 1).toUpperCase()}
-      className={Styles.avatar}
-    />
-  );
+        member!.on(RoomMemberEvent.Name, updateName);
+        return () => {
+          member!.removeListener(RoomMemberEvent.Name, updateName);
+        };
+      }
+    }, [item.member]);
 
-  return (
-    <>
+    // Create an avatar.
+    const avatar = (
+      <Avatar
+        key={item.member?.userId}
+        size={Math.round(Math.min(targetWidth, targetHeight) / 2)}
+        src={item.member?.getMxcAvatarUrl()}
+        fallback={displayName.slice(0, 1).toUpperCase()}
+        className={Styles.avatar}
+      />
+    );
+
+    // Make sure that the tile is draggable and work well within video grid layout.
+    //
+    // Firefox doesn't respect the disablePictureInPicture attribute
+    // https://bugzilla.mozilla.org/show_bug.cgi?id=1611831
+    const tileRef = React.useRef<HTMLElement | null>(null);
+    useDrag((state) => onDragRef?.current!(id, state), {
+      target: tileRef,
+      filterTaps: true,
+      preventScroll: true,
+    });
+
+    return (
       <VideoTile
+        ref={tileRef}
         sfuParticipant={item.sfuParticipant}
         content={item.content}
         name={displayName}
         avatar={avatar}
         {...rest}
       />
-    </>
-  );
-}
+    );
+  }
+);

src/video-grid/model.ts

@@ -0,0 +1,416 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import TinyQueue from "tinyqueue";
+
+import { TileDescriptor } from "./VideoGrid";
+
+/**
+ * A 1×1 cell in a grid which belongs to a tile.
+ */
+export interface Cell {
+  /**
+   * The item displayed on the tile.
+   */
+  item: TileDescriptor<unknown>;
+  /**
+   * Whether this cell is the origin (top left corner) of the tile.
+   */
+  origin: boolean;
+  /**
+   * The width, in columns, of the tile.
+   */
+  columns: number;
+  /**
+   * The height, in rows, of the tile.
+   */
+  rows: number;
+}
+
+export interface Grid {
+  columns: number;
+  /**
+   * The cells of the grid, in left-to-right top-to-bottom order.
+   * undefined = empty.
+   */
+  cells: (Cell | undefined)[];
+}
+
+/**
+ * Gets the paths that tiles should travel along in the grid to reach a
+ * particular destination.
+ * @param dest The destination index.
+ * @param g The grid.
+ * @returns An array in which each cell holds the index of the next cell to move
+ *   to to reach the destination, or null if it is the destination.
+ */
+export function getPaths(dest: number, g: Grid): (number | null)[] {
+  const destRow = row(dest, g);
+  const destColumn = column(dest, g);
+
+  // This is Dijkstra's algorithm
+
+  const distances = new Array<number>(dest + 1).fill(Infinity);
+  distances[dest] = 0;
+  const edges = new Array<number | null | undefined>(dest).fill(undefined);
+  edges[dest] = null;
+  const heap = new TinyQueue([dest], (i) => distances[i]);
+
+  const visit = (curr: number, via: number) => {
+    const viaCell = g.cells[via];
+    const viaLargeTile =
+      viaCell !== undefined && (viaCell.rows > 1 || viaCell.columns > 1);
+    // Since it looks nicer to have paths go around large tiles, we impose an
+    // increased cost for moving through them
+    const distanceVia = distances[via] + (viaLargeTile ? 8 : 1);
+
+    if (distanceVia < distances[curr]) {
+      distances[curr] = distanceVia;
+      edges[curr] = via;
+      heap.push(curr);
+    }
+  };
+
+  while (heap.length > 0) {
+    const via = heap.pop()!;
+    const viaRow = row(via, g);
+    const viaColumn = column(via, g);
+
+    // Visit each neighbor
+    if (viaRow > 0) visit(via - g.columns, via);
+    if (viaColumn > 0) visit(via - 1, via);
+    if (viaColumn < (viaRow === destRow ? destColumn : g.columns - 1))
+      visit(via + 1, via);
+    if (
+      viaRow < destRow - 1 ||
+      (viaRow === destRow - 1 && viaColumn <= destColumn)
+    )
+      visit(via + g.columns, via);
+  }
+
+  // The heap is empty, so we've generated all paths
+  return edges as (number | null)[];
+}
+
+function findLastIndex<T>(
+  array: T[],
+  predicate: (item: T) => boolean
+): number | null {
+  for (let i = array.length - 1; i >= 0; i--) {
+    if (predicate(array[i])) return i;
+  }
+
+  return null;
+}
+
+const findLast1By1Index = (g: Grid): number | null =>
+  findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1);
+
+export function row(index: number, g: Grid): number {
+  return Math.floor(index / g.columns);
+}
+
+export function column(index: number, g: Grid): number {
+  return ((index % g.columns) + g.columns) % g.columns;
+}
+
+function inArea(index: number, start: number, end: number, g: Grid): boolean {
+  const indexColumn = column(index, g);
+  const indexRow = row(index, g);
+  return (
+    indexRow >= row(start, g) &&
+    indexRow <= row(end, g) &&
+    indexColumn >= column(start, g) &&
+    indexColumn <= column(end, g)
+  );
+}
+
+function* cellsInArea(
+  start: number,
+  end: number,
+  g: Grid
+): Generator<number, void, unknown> {
+  const startColumn = column(start, g);
+  const endColumn = column(end, g);
+  for (
+    let i = start;
+    i <= end;
+    i =
+      column(i, g) === endColumn
+        ? i + g.columns + startColumn - endColumn
+        : i + 1
+  )
+    yield i;
+}
+
+export function forEachCellInArea(
+  start: number,
+  end: number,
+  g: Grid,
+  fn: (c: Cell | undefined, i: number) => void
+): void {
+  for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i);
+}
+
+function allCellsInArea(
+  start: number,
+  end: number,
+  g: Grid,
+  fn: (c: Cell | undefined, i: number) => boolean
+): boolean {
+  for (const i of cellsInArea(start, end, g)) {
+    if (!fn(g.cells[i], i)) return false;
+  }
+
+  return true;
+}
+
+const areaEnd = (
+  start: number,
+  columns: number,
+  rows: number,
+  g: Grid
+): number => start + columns - 1 + g.columns * (rows - 1);
+
+/**
+ * Gets the index of the next gap in the grid that should be backfilled by 1×1
+ * tiles.
+ */
+function getNextGap(g: Grid): number | null {
+  const last1By1Index = findLast1By1Index(g);
+  if (last1By1Index === null) return null;
+
+  for (let i = 0; i < last1By1Index; i++) {
+    // To make the backfilling process look natural when there are multiple
+    // gaps, we actually scan each row from right to left
+    const j =
+      (row(i, g) === row(last1By1Index, g)
+        ? last1By1Index
+        : (row(i, g) + 1) * g.columns) -
+      1 -
+      column(i, g);
+
+    if (g.cells[j] === undefined) return j;
+  }
+
+  return null;
+}
+
+/**
+ * Backfill any gaps in the grid.
+ */
+export function fillGaps(g: Grid): Grid {
+  const result: Grid = { ...g, cells: [...g.cells] };
+  let gap = getNextGap(result);
+
+  if (gap !== null) {
+    const pathsToEnd = getPaths(findLast1By1Index(result)!, result);
+
+    do {
+      let filled = false;
+      let to = gap;
+      let from = pathsToEnd[gap];
+
+      // First, attempt to fill the gap by moving 1×1 tiles backwards from the
+      // end of the grid along a set path
+      while (from !== null) {
+        const toCell = result.cells[to];
+        const fromCell = result.cells[from];
+
+        // Skip over slots that are already full
+        if (toCell !== undefined) {
+          to = pathsToEnd[to]!;
+          // Skip over large tiles. Also, we might run into gaps along the path
+          // created during the filling of previous gaps. Skip over those too;
+          // they'll be picked up on the next iteration of the outer loop.
+        } else if (
+          fromCell === undefined ||
+          fromCell.rows > 1 ||
+          fromCell.columns > 1
+        ) {
+          from = pathsToEnd[from];
+        } else {
+          result.cells[to] = result.cells[from];
+          result.cells[from] = undefined;
+          filled = true;
+          to = pathsToEnd[to]!;
+          from = pathsToEnd[from];
+        }
+      }
+
+      // In case the path approach failed, fall back to taking the very last 1×1
+      // tile, and just dropping it into place
+      if (!filled) {
+        const last1By1Index = findLast1By1Index(result)!;
+        result.cells[gap] = result.cells[last1By1Index];
+        result.cells[last1By1Index] = undefined;
+      }
+
+      gap = getNextGap(result);
+    } while (gap !== null);
+  }
+
+  // TODO: If there are any large tiles on the last row, shuffle them back
+  // upwards into a full row
+
+  // Shrink the array to remove trailing gaps
+  const finalLength =
+    (findLastIndex(result.cells, (c) => c !== undefined) ?? -1) + 1;
+  if (finalLength < result.cells.length)
+    result.cells = result.cells.slice(0, finalLength);
+
+  return result;
+}
+
+export function appendItems(items: TileDescriptor<unknown>[], g: Grid): Grid {
+  return {
+    ...g,
+    cells: [
+      ...g.cells,
+      ...items.map((i) => ({
+        item: i,
+        origin: true,
+        columns: 1,
+        rows: 1,
+      })),
+    ],
+  };
+}
+
+/**
+ * Changes the size of a tile, rearranging the grid to make space.
+ * @param tileId The ID of the tile to modify.
+ * @param g The grid.
+ * @returns The updated grid.
+ */
+export function cycleTileSize(tileId: string, g: Grid): Grid {
+  const from = g.cells.findIndex((c) => c?.item.id === tileId);
+  if (from === -1) return g; // Tile removed, no change
+  const fromWidth = g.cells[from]!.columns;
+  const fromHeight = g.cells[from]!.rows;
+  const fromEnd = areaEnd(from, fromWidth, fromHeight, g);
+
+  // The target dimensions, which toggle between 1×1 and larger than 1×1
+  const [toWidth, toHeight] =
+    fromWidth === 1 && fromHeight === 1
+      ? [Math.min(3, Math.max(2, g.columns - 1)), 2]
+      : [1, 1];
+
+  // If we're expanding the tile, we want to create enough new rows at the
+  // tile's target position such that every new unit of grid area created during
+  // the expansion can fit within the new rows.
+  // We do it this way, since it's easier to backfill gaps in the grid than it
+  // is to push colliding tiles outwards.
+  const newRows = Math.max(
+    0,
+    Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns)
+  );
+
+  // This is the grid with the new rows added
+  const gappyGrid: Grid = {
+    ...g,
+    cells: new Array(g.cells.length + newRows * g.columns),
+  };
+
+  // The next task is to scan for a spot to place the modified tile. Since we
+  // might be creating new rows at the target position, this spot can be shorter
+  // than the target height.
+  const candidateWidth = toWidth;
+  const candidateHeight = toHeight - newRows;
+
+  // To make the tile appear to expand outwards from its center, we're actually
+  // scanning for locations to put the *center* of the tile. These numbers are
+  // the offsets between the tile's origin and its center.
+  const scanColumnOffset = Math.floor((toWidth - 1) / 2);
+  const scanRowOffset = Math.floor((toHeight - 1) / 2);
+
+  const nextScanLocations = new Set<number>([from]);
+  const rows = row(g.cells.length - 1, g) + 1;
+  let to: number | null = null;
+
+  // The contents of a given cell are 'displaceable' if it's empty, holds a 1×1
+  // tile, or is part of the original tile we're trying to reposition
+  const displaceable = (c: Cell | undefined, i: number): boolean =>
+    c === undefined ||
+    (c.columns === 1 && c.rows === 1) ||
+    inArea(i, from, fromEnd, g);
+
+  // Do the scanning
+  for (const scanLocation of nextScanLocations) {
+    const start = scanLocation - scanColumnOffset - g.columns * scanRowOffset;
+    const end = areaEnd(start, candidateWidth, candidateHeight, g);
+    const startColumn = column(start, g);
+    const startRow = row(start, g);
+    const endColumn = column(end, g);
+    const endRow = row(end, g);
+
+    if (
+      start >= 0 &&
+      endColumn - startColumn + 1 === candidateWidth &&
+      allCellsInArea(start, end, g, displaceable)
+    ) {
+      // This location works!
+      to = start;
+      break;
+    }
+
+    // Scan outwards in all directions
+    if (startColumn > 0) nextScanLocations.add(scanLocation - 1);
+    if (endColumn < g.columns - 1) nextScanLocations.add(scanLocation + 1);
+    if (startRow > 0) nextScanLocations.add(scanLocation - g.columns);
+    if (endRow < rows - 1) nextScanLocations.add(scanLocation + g.columns);
+  }
+
+  // If there is no space in the grid, give up
+  if (to === null) return g;
+
+  const toRow = row(to, g);
+
+  // Copy tiles from the original grid to the new one, with the new rows
+  // inserted at the target location
+  g.cells.forEach((c, src) => {
+    if (c?.origin && c.item.id !== tileId) {
+      const offset =
+        row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0;
+      forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => {
+        gappyGrid.cells[i + offset] = c;
+      });
+    }
+  });
+
+  // Place the tile in its target position, making a note of the tiles being
+  // overwritten
+  const displacedTiles: Cell[] = [];
+  const toEnd = areaEnd(to, toWidth, toHeight, g);
+  forEachCellInArea(to, toEnd, gappyGrid, (c, i) => {
+    if (c !== undefined) displacedTiles.push(c);
+    gappyGrid.cells[i] = {
+      item: g.cells[from]!.item,
+      origin: i === to,
+      columns: toWidth,
+      rows: toHeight,
+    };
+  });
+
+  // Place the displaced tiles in the remaining space
+  for (let i = 0; displacedTiles.length > 0; i++) {
+    if (gappyGrid.cells[i] === undefined)
+      gappyGrid.cells[i] = displacedTiles.shift();
+  }
+
+  // Fill any gaps that remain
+  return fillGaps(gappyGrid);
+}