Add docker-compose & helm config

- Add docker-compose as well as instructions to run it locally with
  exposed ports 8080 & 8443 (with required nginx.conf additions)
- Add helm chart; it's pretty default and contains one template extension:
  we should be able to mount public/config.json from values.yaml (untested)

.github/workflows/netlify-livekit.yaml

@@ -0,0 +1,88 @@
+name: Netlify LiveKit Experiment
+on:
+  workflow_run:
+    workflows: ["Build"]
+    types:
+      - completed
+    branches:
+      - "livekit-experiment"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      deployments: write
+    # Important: the 'branches' filter above will match the 'livekit-experiment' branch on forks,
+    # so we need to check the head repo too in order to not run on PRs from forks
+    # We check the branch name again too just for completeness
+    # (Is there a nicer way to see if a PR is from a fork?)
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name == 'vector-im/element-call' && github.event.workflow_run.head_branch == 'livekit-experiment'
+    steps:
+      - name: Create Deployment
+        uses: bobheadxi/deployments@v1
+        id: deployment
+        with:
+          step: start
+          token: ${{ secrets.GITHUB_TOKEN }}
+          env: livekit-experiment-branch-cd
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: "Download artifact"
+        uses: actions/github-script@v3.1.0
+        with:
+          script: |
+            const artifacts = await github.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: ${{ github.event.workflow_run.id }},
+            });
+            const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == "build"
+            })[0];
+            const download = await github.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+            });
+            const fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/build.zip', Buffer.from(download.data));
+
+      - name: Extract Artifacts
+        run: unzip -d dist build.zip && rm build.zip
+
+      - name: Add redirects file
+        # We fetch from github directly as we don't bother checking out the repo
+        run: curl -s https://raw.githubusercontent.com/vector-im/element-call/livekit-experiment/config/netlify_redirects > dist/_redirects
+
+      - name: Add config file
+        run: curl -s https://raw.githubusercontent.com/vector-im/element-call/livekit-experiment/config/element_io_preview.json > dist/config.json
+
+      - name: Deploy to Netlify
+        id: netlify
+        uses: nwtgck/actions-netlify@v1.2.3
+        with:
+          publish-dir: dist
+          deploy-message: "Deploy from GitHub Actions"
+          production-branch: livekit-experiment
+          production-deploy: true
+          # These don't work because we're in workflow_run
+          enable-pull-request-comment: false
+          enable-commit-comment: false
+          github-deployment-environment: livekit-experiment
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: e3b9fa82-c040-4db6-b4bf-42b524d57423
+        timeout-minutes: 1
+
+      - name: Update deployment status
+        uses: bobheadxi/deployments@v1
+        if: always()
+        with:
+          step: finish
+          override: false
+          token: ${{ secrets.GITHUB_TOKEN }}
+          status: ${{ job.status }}
+          env: ${{ steps.deployment.outputs.env }}
+          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
+          env_url: ${{ steps.netlify.outputs.deploy-url }}

.github/workflows/test.yaml

@@ -1,6 +1,8 @@
 name: Run jest tests
 on:
   pull_request: {}
+  push:
+    branches: [livekit]
 jobs:
   jest:
     name: Run jest tests
@@ -18,4 +20,5 @@ jobs:
         run: "yarn run test"
       - name: Upload to codecov
         uses: codecov/codecov-action@v3
-        flags: unittests
+        with:
+          flags: unittests

.gitignore

@@ -7,3 +7,6 @@ dist-ssr
 .idea/
 public/config.json
 /coverage
+.*.sw?
+
+certs/

README.md

@@ -90,6 +90,12 @@ You're now ready to launch the development server:
 yarn dev
 ```
 
+Alternatively, you could run it in docker-compose:
+
+```
+yarn build ; docker-compose build; docker-compose up -d
+```
+
 ## Configuration
 
 There are currently two different config files. `.env` holds variables that are used at build time, while `public/config.json` holds variables that are used at runtime. Documentation and default values for `public/config.json` can be found in [ConfigOptions.ts](src/config/ConfigOptions.ts).

docker-compose.yaml

@@ -0,0 +1,23 @@
+# LiveKit requires host networking, which is only available on Linux
+# This compose will not function correctly on Mac or Windows
+version: "3.9"
+
+networks:
+  lkbackend:
+
+services:
+  call:
+    build:
+      context: ./
+    container_name: element-call-backend
+    ports:
+      - 8080:8080
+      - 8443:8443
+    deploy:
+      restart_policy:
+        condition: on-failure
+    networks:
+      - lkbackend
+    volumes:
+      - ./certs/element-call.crt:/etc/ssl/certs/element-call.crt
+      - ./certs/element-call.key:/etc/ssl/private/element-call.key

helm-chart/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm-chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: element-call
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.16.0"

helm-chart/application.yml

@@ -0,0 +1,32 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ungleich-matrix-element-call
+  namespace: argocd
+  finalizers:
+  - resources-finalizer.argocd.argoproj.io
+spec:
+  destination:
+    namespace: ungleich
+    server: 'https://kubernetes.default.svc'
+  source:
+    path: apps/prod/element-call
+    repoURL: 'https://code.ungleich.ch/ungleich-intern/k8s-config.git'
+    targetRevision: HEAD
+    helm:
+      parameters:
+        - name: storage.letsencrypt.storageClass
+          value: rook-ceph-block-hdd
+        - name: storage.letsencrypt.size
+          value: 60Mi
+        - name: letsencryptStaging
+          value: 'no'
+        - name: fqdn
+          value: 'call.ungleich.ch'
+  project: default
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

helm-chart/templates/NOTES.txt

@@ -0,0 +1,22 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "element-call.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "element-call.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "element-call.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "element-call.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

helm-chart/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "element-call.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "element-call.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "element-call.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "element-call.labels" -}}
+helm.sh/chart: {{ include "element-call.chart" . }}
+{{ include "element-call.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "element-call.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "element-call.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "element-call.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "element-call.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

helm-chart/templates/configmap.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: app-config
+  namespace: default
+data:
+  {{ .Values.config.fileName }}: |
+    {{ .Values.config.data  | toPrettyJson |  quote }}

helm-chart/templates/deployment.yaml

@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "element-call.fullname" . }}
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "element-call.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "element-call.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "element-call.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: {{ .Values.service.port }}
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          volumeMounts:
+          - name: config-volume
+            mountPath: "{{ .Values.config.path }}"
+            subPath: "{{ .Values.config.fileName }}"
+      volumes:
+        - name: config-volume
+          configMap:
+            name: app-config
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}

helm-chart/templates/hpa.yaml

@@ -0,0 +1,28 @@
+{{- if .Values.autoscaling.enabled }}
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ include "element-call.fullname" . }}
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ include "element-call.fullname" . }}
+  minReplicas: {{ .Values.autoscaling.minReplicas }}
+  maxReplicas: {{ .Values.autoscaling.maxReplicas }}
+  metrics:
+    {{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: cpu
+        targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
+    {{- end }}
+    {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    - type: Resource
+      resource:
+        name: memory
+        targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
+    {{- end }}
+{{- end }}

helm-chart/templates/ingress.yaml

@@ -0,0 +1,61 @@
+{{- if .Values.ingress.enabled -}}
+{{- $fullName := include "element-call.fullname" . -}}
+{{- $svcPort := .Values.service.port -}}
+{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+  {{- with .Values.ingress.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.className }}
+  {{- end }}
+  {{- if .Values.ingress.tls }}
+  tls:
+    {{- range .Values.ingress.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ $fullName }}
+                port:
+                  number: {{ $svcPort }}
+              {{- else }}
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}

helm-chart/templates/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "element-call.fullname" . }}
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    {{- include "element-call.selectorLabels" . | nindent 4 }}

helm-chart/templates/serviceaccount.yaml

@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "element-call.serviceAccountName" . }}
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}

helm-chart/templates/tests/test-connection.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ include "element-call.fullname" . }}-test-connection"
+  labels:
+    {{- include "element-call.labels" . | nindent 4 }}
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['{{ include "element-call.fullname" . }}:{{ .Values.service.port }}']
+  restartPolicy: Never

helm-chart/values.yaml

@@ -0,0 +1,89 @@
+# Default values for element-call.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+
+image:
+  repository: element-call_call
+  pullPolicy: Never
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: false
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: NodePort
+  port: 8080
+  targetPort: 8080
+  nodePort: 30070
+
+ingress:
+  enabled: false
+  className: ""
+  annotations: {}
+    # kubernetes.io/ingress.class: nginx
+    # kubernetes.io/tls-acme: "true"
+  hosts:
+    - host: chart-example.local
+      paths:
+        - path: /
+          pathType: ImplementationSpecific
+  tls: []
+  #  - secretName: chart-example-tls
+  #    hosts:
+  #      - chart-example.local
+
+autoscaling:
+  enabled: false
+  minReplicas: 1
+  maxReplicas: 3
+  targetCPUUtilizationPercentage: 80
+  # targetMemoryUtilizationPercentage: 80
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+
+config:
+  path: /app/public/
+  fileName: config.json
+  data: |
+         {
+           "default_server_config": {
+             "m.homeserver": {
+                 "base_url": "https://ungleich.matrix.ungleich.cloud",
+                 "server_name": "ungleich.ch"
+             }
+           },
+           "features": {
+             "feature_group_calls_without_video_and_audio": true
+           }
+         }

package.json

@@ -53,7 +53,7 @@
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f884c78579c336a03bc20ff8f4e92c46582822b6",
     "matrix-widget-api": "^1.3.1",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",

public/locales/en-GB/app.json

@@ -36,6 +36,7 @@
   "Close": "Close",
   "Confirm password": "Confirm password",
   "Connection lost": "Connection lost",
+  "Connectivity to the server has been lost.": "Connectivity to the server has been lost.",
   "Copied!": "Copied!",
   "Copy": "Copy",
   "Copy and share this call link": "Copy and share this call link",

src/App.tsx

@@ -29,6 +29,7 @@ import { usePageFocusStyle } from "./usePageFocusStyle";
 import { SequenceDiagramViewerPage } from "./SequenceDiagramViewerPage";
 import { InspectorContextProvider } from "./room/GroupCallInspector";
 import { CrashView, LoadingView } from "./FullScreenView";
+import { DisconnectedBanner } from "./DisconnectedBanner";
 import { Initializer } from "./initializer";
 import { MediaHandlerProvider } from "./settings/useMediaHandler";
 
@@ -60,6 +61,7 @@ export default function App({ history }: AppProps) {
               <InspectorContextProvider>
                 <Sentry.ErrorBoundary fallback={errorPage}>
                   <OverlayProvider>
+                    <DisconnectedBanner />
                     <Switch>
                       <SentryRoute exact path="/">
                         <HomePage />

src/ClientContext.tsx

@@ -25,9 +25,10 @@ import React, {
   useRef,
 } from "react";
 import { useHistory } from "react-router-dom";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
 import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
+import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
 
 import { ErrorView } from "./FullScreenView";
 import {
@@ -70,6 +71,8 @@ const loadSession = (): Session => {
 const saveSession = (session: Session) =>
   localStorage.setItem("matrix-auth-store", JSON.stringify(session));
 const clearSession = () => localStorage.removeItem("matrix-auth-store");
+const isDisconnected = (syncState, syncData) =>
+  syncState === "ERROR" && syncData?.error?.name === "ConnectionError";
 
 interface ClientState {
   loading: boolean;
@@ -81,6 +84,7 @@ interface ClientState {
   logout: () => void;
   setClient: (client: MatrixClient, session: Session) => void;
   error?: Error;
+  disconnected: boolean;
 }
 
 const ClientContext = createContext<ClientState>(null);
@@ -98,7 +102,15 @@ export const ClientProvider: FC<Props> = ({ children }) => {
   const history = useHistory();
   const initializing = useRef(false);
   const [
-    { loading, isAuthenticated, isPasswordlessUser, client, userName, error },
+    {
+      loading,
+      isAuthenticated,
+      isPasswordlessUser,
+      client,
+      userName,
+      error,
+      disconnected,
+    },
     setState,
   ] = useState<ClientProviderState>({
     loading: true,
@@ -107,8 +119,18 @@ export const ClientProvider: FC<Props> = ({ children }) => {
     client: undefined,
     userName: null,
     error: undefined,
+    disconnected: false,
   });
 
+  const onSync = (state: SyncState, _old: SyncState, data: ISyncStateData) => {
+    setState((currentState) => {
+      const disconnected = isDisconnected(state, data);
+      return disconnected === currentState.disconnected
+        ? currentState
+        : { ...currentState, disconnected };
+    });
+  };
+
   useEffect(() => {
     // In case the component is mounted, unmounted, and remounted quickly (as
     // React does in strict mode), we need to make sure not to doubly initialize
@@ -183,9 +205,10 @@ export const ClientProvider: FC<Props> = ({ children }) => {
         }
       }
     };
-
+    let clientWithListener: MatrixClient;
     init()
       .then(({ client, isPasswordlessUser }) => {
+        clientWithListener = client;
         setState({
           client,
           loading: false,
@@ -193,7 +216,12 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser,
           userName: client?.getUserIdLocalpart(),
           error: undefined,
+          disconnected: isDisconnected(
+            client?.getSyncState,
+            client?.getSyncStateData
+          ),
         });
+        clientWithListener?.on(ClientEvent.Sync, onSync);
       })
       .catch((err) => {
         logger.error(err);
@@ -204,9 +232,13 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: false,
           userName: null,
           error: undefined,
+          disconnected: false,
         });
       })
       .finally(() => (initializing.current = false));
+    return () => {
+      clientWithListener?.removeListener(ClientEvent.Sync, onSync);
+    };
   }, []);
 
   const changePassword = useCallback(
@@ -235,6 +267,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
         isPasswordlessUser: false,
         userName: client.getUserIdLocalpart(),
         error: undefined,
+        disconnected: false,
       });
     },
     [client]
@@ -256,6 +289,10 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: session.passwordlessUser,
           userName: newClient.getUserIdLocalpart(),
           error: undefined,
+          disconnected: isDisconnected(
+            newClient.getSyncState(),
+            newClient.getSyncStateData()
+          ),
         });
       } else {
         clearSession();
@@ -267,6 +304,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: false,
           userName: null,
           error: undefined,
+          disconnected: false,
         });
       }
     },
@@ -284,6 +322,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       isPasswordlessUser: true,
       userName: "",
       error: undefined,
+      disconnected: false,
     });
     history.push("/");
     PosthogAnalytics.instance.setRegistrationType(RegistrationType.Guest);
@@ -326,6 +365,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       userName,
       setClient,
       error: undefined,
+      disconnected,
     }),
     [
       loading,
@@ -336,6 +376,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       logout,
       userName,
       setClient,
+      disconnected,
     ]
   );
 

src/DisconnectedBanner.module.css

@@ -0,0 +1,27 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.banner {
+  position: absolute;
+  padding: 29px;
+  background-color: var(--quaternary-content);
+  vertical-align: middle;
+  font-size: var(--font-size-body);
+  text-align: center;
+  z-index: 1;
+  top: 76px;
+  width: calc(100% - 58px);
+}

src/DisconnectedBanner.tsx

@@ -0,0 +1,47 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import classNames from "classnames";
+import React, { HTMLAttributes, ReactNode } from "react";
+import { useTranslation } from "react-i18next";
+
+import styles from "./DisconnectedBanner.module.css";
+import { useClient } from "./ClientContext";
+
+interface DisconnectedBannerProps extends HTMLAttributes<HTMLElement> {
+  children?: ReactNode;
+  className?: string;
+}
+
+export function DisconnectedBanner({
+  children,
+  className,
+  ...rest
+}: DisconnectedBannerProps) {
+  const { t } = useTranslation();
+  const { disconnected } = useClient();
+
+  return (
+    <>
+      {disconnected && (
+        <div className={classNames(styles.banner, className)} {...rest}>
+          {children}
+          {t("Connectivity to the server has been lost.")}
+        </div>
+      )}
+    </>
+  );
+}

src/analytics/PosthogAnalytics.ts

@@ -70,6 +70,7 @@ export enum RegistrationType {
 interface PlatformProperties {
   appVersion: string;
   matrixBackend: "embedded" | "jssdk";
+  callBackend: "livekit" | "full-mesh";
 }
 
 interface PosthogSettings {
@@ -191,6 +192,7 @@ export class PosthogAnalytics {
     return {
       appVersion,
       matrixBackend: widget ? "embedded" : "jssdk",
+      callBackend: "full-mesh",
     };
   }
 

src/array-utils.ts

@@ -0,0 +1,38 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/**
+ * Gets the index of the last element in the array to satsify the given
+ * predicate.
+ */
+// TODO: remove this once TypeScript recognizes the existence of
+// Array.prototype.findLastIndex
+export function findLastIndex<T>(
+  array: T[],
+  predicate: (item: T) => boolean
+): number | null {
+  for (let i = array.length - 1; i >= 0; i--) {
+    if (predicate(array[i])) return i;
+  }
+
+  return null;
+}
+
+/**
+ * Counts the number of elements in an array that satsify the given predicate.
+ */
+export const count = <T>(array: T[], predicate: (item: T) => boolean): number =>
+  array.reduce((acc, item) => (predicate(item) ? acc + 1 : acc), 0);

src/initializer.tsx

@@ -45,6 +45,12 @@ class DependencyLoadStates {
 
 export class Initializer {
   private static internalInstance: Initializer;
+  private isInitialized = false;
+
+  public static isInitialized(): boolean {
+    return Initializer.internalInstance?.isInitialized;
+  }
+
   public static initBeforeReact() {
     // this maybe also needs to return a promise in the future,
     // if we have to do async inits before showing the loading screen
@@ -223,6 +229,7 @@ export class Initializer {
     if (this.loadStates.allDepsAreLoaded()) {
       // resolve if there is no dependency that is not loaded
       resolve();
+      this.isInitialized = true;
     }
   }
   private initPromise: Promise<void> | null;

src/matrix-utils.ts

@@ -61,11 +61,11 @@ function waitForSync(client: MatrixClient) {
       data: ISyncStateData
     ) => {
       if (state === "PREPARED") {
+        client.removeListener(ClientEvent.Sync, onSync);
         resolve();
-        client.removeListener(ClientEvent.Sync, onSync);
       } else if (state === "ERROR") {
-        reject(data?.error);
         client.removeListener(ClientEvent.Sync, onSync);
+        reject(data?.error);
       }
     };
     client.on(ClientEvent.Sync, onSync);

src/room/InCallView.tsx

@@ -216,6 +216,8 @@ export function InCallView({
           focused: screenshareFeeds.length === 0 && callFeed === activeSpeaker,
           isLocal: member.userId === localUserId && deviceId === localDeviceId,
           presenter,
+          isSpeaker: callFeed === activeSpeaker,
+          largeBaseSize: false,
           connectionState,
         });
       }
@@ -228,6 +230,7 @@ export function InCallView({
     // Add the screenshares too
     for (const screenshareFeed of screenshareFeeds) {
       const member = screenshareFeed.getMember()!;
+      const deviceId = screenshareFeed.deviceId!;
       const connectionState = participants
         .get(member)
         ?.get(screenshareFeed.deviceId!)?.connectionState;
@@ -242,6 +245,9 @@ export function InCallView({
           focused: true,
           isLocal: screenshareFeed.isLocal(),
           presenter: false,
+          isSpeaker: screenshareFeed === activeSpeaker,
+          largeBaseSize: true,
+          placeNear: `${member.userId} ${deviceId}`,
           connectionState,
         });
       }

src/settings/submit-rageshake.ts

@@ -101,6 +101,7 @@ export function useSubmitRageshake(): {
         body.append("user_agent", userAgent);
         body.append("installed_pwa", "false");
         body.append("touch_input", touchInput);
+        body.append("call_backend", "full-mesh");
 
         if (client) {
           const userId = client.getUserId();

src/video-grid/NewVideoGrid.module.css

@@ -18,7 +18,7 @@ limitations under the License.
   contain: strict;
   position: relative;
   flex-grow: 1;
-  padding: 0 20px;
+  padding: 0 20px var(--footerHeight);
   overflow-y: auto;
   overflow-x: hidden;
 }
@@ -28,7 +28,6 @@ limitations under the License.
   display: grid;
   grid-auto-rows: 163px;
   gap: 8px;
-  padding-bottom: var(--footerHeight);
 }
 
 .slot {
@@ -37,7 +36,7 @@ limitations under the License.
 
 @media (min-width: 800px) {
   .grid {
-    padding: 0 22px;
+    padding: 0 22px var(--footerHeight);
   }
 
   .slotGrid {

src/video-grid/NewVideoGrid.tsx

@@ -43,7 +43,10 @@ import {
   fillGaps,
   forEachCellInArea,
   cycleTileSize,
-  appendItems,
+  addItems,
+  tryMoveTile,
+  resize,
+  promoteSpeakers,
 } from "./model";
 import { TileWrapper } from "./TileWrapper";
 
@@ -81,15 +84,21 @@ const useGridState = (
         }),
       };
 
-      // Step 2: Backfill gaps left behind by removed tiles
-      const grid2 = fillGaps(grid1);
+      // Step 2: Resize the grid if necessary and backfill gaps left behind by
+      // removed tiles
+      // Resizing already takes care of backfilling gaps
+      const grid2 =
+        columns !== grid1.columns ? resize(grid1, columns!) : fillGaps(grid1);
 
       // Step 3: Add new tiles to the end of the grid
       const existingItemIds = new Set(
         grid2.cells.filter((c) => c !== undefined).map((c) => c!.item.id)
       );
       const newItems = items.filter((i) => !existingItemIds.has(i.id));
-      const grid3 = appendItems(newItems, grid2);
+      const grid3 = addItems(newItems, grid2);
+
+      // Step 4: Promote speakers to the top
+      promoteSpeakers(grid3);
 
       return { ...grid3, generation: prevGrid.generation + 1 };
     },
@@ -204,14 +213,10 @@ export const NewVideoGrid: FC<Props> = ({
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [slotGrid, slotGridGeneration, gridBounds]);
 
-  const [columns] = useReactiveState<number | null>(
-    // Since grid resizing isn't implemented yet, pick a column count on mount
-    // and stick to it
-    (prevColumns) =>
-      prevColumns !== undefined && prevColumns !== null
-        ? prevColumns
-        : // The grid bounds might not be known yet
-        gridBounds.width === 0
+  const columns = useMemo(
+    () =>
+      // The grid bounds might not be known yet
+      gridBounds.width === 0
         ? null
         : Math.max(2, Math.floor(gridBounds.width * 0.0045)),
     [gridBounds]
@@ -257,7 +262,7 @@ export const NewVideoGrid: FC<Props> = ({
       enter: { opacity: 1, scale: 1, immediate: disableAnimations },
       update: ({ item, x, y, width, height }: Tile) =>
         item.id === dragState.current?.tileId
-          ? {}
+          ? null
           : {
               x,
               y,
@@ -280,57 +285,68 @@ export const NewVideoGrid: FC<Props> = ({
   const animateDraggedTile = (endOfGesture: boolean) => {
     const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!;
     const tile = tiles.find((t) => t.item.id === tileId)!;
+    const originIndex = grid!.cells.findIndex((c) => c?.item.id === tileId);
+    const originCell = grid!.cells[originIndex]!;
 
-    springRef.start((_i, controller) => {
-      if ((controller.item as Tile).item.id === tileId) {
-        if (endOfGesture) {
-          return {
-            scale: 1,
-            zIndex: 1,
-            shadow: 1,
-            x: tile.x,
-            y: tile.y,
-            width: tile.width,
-            height: tile.height,
-            immediate: disableAnimations || ((key) => key === "zIndex"),
-            // Allow the tile's position to settle before pushing its
-            // z-index back down
-            delay: (key) => (key === "zIndex" ? 500 : 0),
-          };
-        } else {
-          return {
-            scale: 1.1,
-            zIndex: 2,
-            shadow: 15,
-            x: tileX,
-            y: tileY,
-            immediate:
-              disableAnimations ||
-              ((key) => key === "zIndex" || key === "x" || key === "y"),
-          };
-        }
-      } else {
-        return {};
-      }
-    });
+    springRef.current
+      .find((c) => (c.item as Tile).item.id === tileId)
+      ?.start(
+        endOfGesture
+          ? {
+              scale: 1,
+              zIndex: 1,
+              shadow: 1,
+              x: tile.x,
+              y: tile.y,
+              width: tile.width,
+              height: tile.height,
+              immediate: disableAnimations || ((key) => key === "zIndex"),
+              // Allow the tile's position to settle before pushing its
+              // z-index back down
+              delay: (key) => (key === "zIndex" ? 500 : 0),
+            }
+          : {
+              scale: 1.1,
+              zIndex: 2,
+              shadow: 15,
+              x: tileX,
+              y: tileY,
+              immediate:
+                disableAnimations ||
+                ((key) => key === "zIndex" || key === "x" || key === "y"),
+            }
+      );
 
-    const overTile = tiles.find(
-      (t) =>
-        cursorX >= t.x &&
-        cursorX < t.x + t.width &&
-        cursorY >= t.y &&
-        cursorY < t.y + t.height
+    const columns = grid!.columns;
+    const rows = row(grid!.cells.length - 1, grid!) + 1;
+
+    const cursorColumn = Math.floor(
+      (cursorX / slotGrid!.clientWidth) * columns
     );
-    if (overTile !== undefined && overTile.item.id !== tileId) {
-      setGrid((g) => ({
-        ...g!,
-        cells: g!.cells.map((c) => {
-          if (c?.item === overTile.item) return { ...c, item: tile.item };
-          if (c?.item === tile.item) return { ...c, item: overTile.item };
-          return c;
-        }),
-      }));
-    }
+    const cursorRow = Math.floor((cursorY / slotGrid!.clientHeight) * rows);
+
+    const cursorColumnOnTile = Math.floor(
+      ((cursorX - tileX) / tile.width) * originCell.columns
+    );
+    const cursorRowOnTile = Math.floor(
+      ((cursorY - tileY) / tile.height) * originCell.rows
+    );
+
+    const dest =
+      Math.max(
+        0,
+        Math.min(
+          columns - originCell.columns,
+          cursorColumn - cursorColumnOnTile
+        )
+      ) +
+      grid!.columns *
+        Math.max(
+          0,
+          Math.min(rows - originCell.rows, cursorRow - cursorRowOnTile)
+        );
+
+    if (dest !== originIndex) setGrid((g) => tryMoveTile(g, originIndex, dest));
   };
 
   // Callback for useDrag. We could call useDrag here, but the default

src/video-grid/TileDescriptor.tsx

@@ -26,7 +26,10 @@ export interface TileDescriptor {
   member: RoomMember;
   focused: boolean;
   presenter: boolean;
+  isSpeaker: boolean;
   callFeed?: CallFeed;
   isLocal?: boolean;
+  largeBaseSize: boolean;
+  placeNear?: string;
   connectionState: ConnectionState;
 }

src/video-grid/model.ts

@@ -17,6 +17,7 @@ limitations under the License.
 import TinyQueue from "tinyqueue";
 
 import { TileDescriptor } from "./TileDescriptor";
+import { count, findLastIndex } from "../array-utils";
 
 /**
  * A 1×1 cell in a grid which belongs to a tile.
@@ -105,17 +106,6 @@ export function getPaths(dest: number, g: Grid): (number | null)[] {
   return edges as (number | null)[];
 }
 
-function findLastIndex<T>(
-  array: T[],
-  predicate: (item: T) => boolean
-): number | null {
-  for (let i = array.length - 1; i >= 0; i--) {
-    if (predicate(array[i])) return i;
-  }
-
-  return null;
-}
-
 const findLast1By1Index = (g: Grid): number | null =>
   findLastIndex(g.cells, (c) => c?.rows === 1 && c?.columns === 1);
 
@@ -185,6 +175,8 @@ const areaEnd = (
   g: Grid
 ): number => start + columns - 1 + g.columns * (rows - 1);
 
+const cloneGrid = (g: Grid): Grid => ({ ...g, cells: [...g.cells] });
+
 /**
  * Gets the index of the next gap in the grid that should be backfilled by 1×1
  * tiles.
@@ -209,11 +201,150 @@ function getNextGap(g: Grid): number | null {
   return null;
 }
 
+/**
+ * Gets the index of the origin of the tile to which the given cell belongs.
+ */
+function getOrigin(g: Grid, index: number): number {
+  const initialColumn = column(index, g);
+
+  for (
+    let i = index;
+    i >= 0;
+    i = column(i, g) === 0 ? i - g.columns + initialColumn : i - 1
+  ) {
+    const cell = g.cells[i];
+    if (
+      cell !== undefined &&
+      cell.origin &&
+      inArea(index, i, areaEnd(i, cell.columns, cell.rows, g), g)
+    )
+      return i;
+  }
+
+  throw new Error("Tile is broken");
+}
+
+/**
+ * Moves the tile at index "from" over to index "to", displacing other tiles
+ * along the way.
+ * Precondition: the destination area must consist of only 1×1 tiles.
+ */
+function moveTile(g: Grid, from: number, to: number) {
+  const tile = g.cells[from]!;
+  const fromEnd = areaEnd(from, tile.columns, tile.rows, g);
+  const toEnd = areaEnd(to, tile.columns, tile.rows, g);
+
+  const displacedTiles: Cell[] = [];
+  forEachCellInArea(to, toEnd, g, (c, i) => {
+    if (c !== undefined && !inArea(i, from, fromEnd, g)) displacedTiles.push(c);
+  });
+
+  const movingCells: Cell[] = [];
+  forEachCellInArea(from, fromEnd, g, (c, i) => {
+    movingCells.push(c!);
+    g.cells[i] = undefined;
+  });
+
+  forEachCellInArea(
+    to,
+    toEnd,
+    g,
+    (_c, i) => (g.cells[i] = movingCells.shift())
+  );
+  forEachCellInArea(
+    from,
+    fromEnd,
+    g,
+    (_c, i) => (g.cells[i] ??= displacedTiles.shift())
+  );
+}
+
+/**
+ * Moves the tile at index "from" over to index "to", if there is space.
+ */
+export function tryMoveTile(g: Grid, from: number, to: number): Grid {
+  const tile = g.cells[from]!;
+
+  if (
+    to >= 0 &&
+    to < g.cells.length &&
+    column(to, g) <= g.columns - tile.columns
+  ) {
+    const fromEnd = areaEnd(from, tile.columns, tile.rows, g);
+    const toEnd = areaEnd(to, tile.columns, tile.rows, g);
+
+    // The contents of a given cell are 'displaceable' if it's empty, holds a
+    // 1×1 tile, or is part of the original tile we're trying to reposition
+    const displaceable = (c: Cell | undefined, i: number): boolean =>
+      c === undefined ||
+      (c.columns === 1 && c.rows === 1) ||
+      inArea(i, from, fromEnd, g);
+
+    if (allCellsInArea(to, toEnd, g, displaceable)) {
+      // The target space is free; move
+      const gClone = cloneGrid(g);
+      moveTile(gClone, from, to);
+      return gClone;
+    }
+  }
+
+  // The target space isn't free; don't move
+  return g;
+}
+
+/**
+ * Attempts to push a tile upwards by one row, displacing 1×1 tiles and shifting
+ * enlarged tiles around when necessary.
+ * @returns Whether the tile was actually pushed
+ */
+function pushTileUp(g: Grid, from: number): boolean {
+  const tile = g.cells[from]!;
+
+  // TODO: pushing large tiles sideways might be more successful in some
+  // situations
+  const cellsAboveAreDisplacable =
+    from - g.columns >= 0 &&
+    allCellsInArea(
+      from - g.columns,
+      from - g.columns + tile.columns - 1,
+      g,
+      (c, i) =>
+        c === undefined ||
+        (c.columns === 1 && c.rows === 1) ||
+        pushTileUp(g, getOrigin(g, i))
+    );
+
+  if (cellsAboveAreDisplacable) {
+    moveTile(g, from, from - g.columns);
+    return true;
+  } else {
+    return false;
+  }
+}
+
 /**
  * Backfill any gaps in the grid.
  */
 export function fillGaps(g: Grid): Grid {
-  const result: Grid = { ...g, cells: [...g.cells] };
+  const result = cloneGrid(g);
+
+  // This will hopefully be the size of the grid after we're done here, assuming
+  // that we can pack the large tiles tightly enough
+  const idealLength = count(result.cells, (c) => c !== undefined);
+
+  // Step 1: Take any large tiles hanging off the bottom of the grid, and push
+  // them upwards
+  for (let i = result.cells.length - 1; i >= idealLength; i--) {
+    const cell = result.cells[i];
+    if (cell !== undefined && (cell.columns > 1 || cell.rows > 1)) {
+      const originIndex =
+        i - (cell.columns - 1) - result.columns * (cell.rows - 1);
+      // If it's not possible to pack the large tiles any tighter, give up
+      if (!pushTileUp(result, originIndex)) break;
+    }
+  }
+
+  // Step 2: Fill all 1×1 gaps
   let gap = getNextGap(result);
 
   if (gap !== null) {
@@ -263,9 +394,6 @@ export function fillGaps(g: Grid): Grid {
     } while (gap !== null);
   }
 
-  // TODO: If there are any large tiles on the last row, shuffle them back
-  // upwards into a full row
-
   // Shrink the array to remove trailing gaps
   const finalLength =
     (findLastIndex(result.cells, (c) => c !== undefined) ?? -1) + 1;
@@ -275,21 +403,104 @@ export function fillGaps(g: Grid): Grid {
   return result;
 }
 
-export function appendItems(items: TileDescriptor[], g: Grid): Grid {
-  return {
-    ...g,
-    cells: [
-      ...g.cells,
-      ...items.map((i) => ({
-        item: i,
-        origin: true,
-        columns: 1,
-        rows: 1,
-      })),
-    ],
+function createRows(g: Grid, count: number, atRow: number): Grid {
+  const result = {
+    columns: g.columns,
+    cells: new Array(g.cells.length + g.columns * count),
   };
+  const offsetAfterNewRows = g.columns * count;
+
+  // Copy tiles from the original grid to the new one, with the new rows
+  // inserted at the target location
+  g.cells.forEach((c, from) => {
+    if (c?.origin) {
+      const offset = row(from, g) >= atRow ? offsetAfterNewRows : 0;
+      forEachCellInArea(
+        from,
+        areaEnd(from, c.columns, c.rows, g),
+        g,
+        (c, i) => {
+          result.cells[i + offset] = c;
+        }
+      );
+    }
+  });
+
+  return result;
 }
 
+/**
+ * Adds a set of new items into the grid.
+ */
+export function addItems(items: TileDescriptor[], g: Grid): Grid {
+  let result = cloneGrid(g);
+
+  for (const item of items) {
+    const cell = {
+      item,
+      origin: true,
+      columns: 1,
+      rows: 1,
+    };
+
+    let placeAt: number;
+    let hasGaps: boolean;
+
+    if (item.placeNear === undefined) {
+      // This item has no special placement requests, so let's put it
+      // uneventfully at the end of the grid
+      placeAt = result.cells.length;
+      hasGaps = false;
+    } else {
+      // This item wants to be placed near another; let's put it on a row
+      // directly below the related tile
+      const placeNear = result.cells.findIndex(
+        (c) => c?.item.id === item.placeNear
+      );
+      if (placeNear === -1) {
+        // Can't find the related tile, so let's give up and place it at the end
+        placeAt = result.cells.length;
+        hasGaps = false;
+      } else {
+        const placeNearCell = result.cells[placeNear]!;
+        const placeNearEnd = areaEnd(
+          placeNear,
+          placeNearCell.columns,
+          placeNearCell.rows,
+          result
+        );
+
+        result = createRows(result, 1, row(placeNearEnd, result) + 1);
+        placeAt =
+          placeNear +
+          Math.floor(placeNearCell.columns / 2) +
+          result.columns * placeNearCell.rows;
+        hasGaps = true;
+      }
+    }
+
+    result.cells[placeAt] = cell;
+
+    if (item.largeBaseSize) {
+      // Cycle the tile size once to set up the tile with its larger base size
+      // This also fills any gaps in the grid, hence no extra call to fillGaps
+      result = cycleTileSize(item.id, result);
+    } else if (hasGaps) {
+      result = fillGaps(result);
+    }
+  }
+
+  return result;
+}
+
+const largeTileDimensions = (g: Grid): [number, number] => [
+  Math.min(3, Math.max(2, g.columns - 1)),
+  2,
+];
+
+const extraLargeTileDimensions = (g: Grid): [number, number] =>
+  g.columns > 3 ? [4, 3] : [g.columns, 2];
+
 /**
  * Changes the size of a tile, rearranging the grid to make space.
  * @param tileId The ID of the tile to modify.
@@ -299,15 +510,19 @@ export function appendItems(items: TileDescriptor[], g: Grid): Grid {
 export function cycleTileSize(tileId: string, g: Grid): Grid {
   const from = g.cells.findIndex((c) => c?.item.id === tileId);
   if (from === -1) return g; // Tile removed, no change
-  const fromWidth = g.cells[from]!.columns;
-  const fromHeight = g.cells[from]!.rows;
+  const fromCell = g.cells[from]!;
+  const fromWidth = fromCell.columns;
+  const fromHeight = fromCell.rows;
   const fromEnd = areaEnd(from, fromWidth, fromHeight, g);
 
-  // The target dimensions, which toggle between 1×1 and larger than 1×1
+  const [baseDimensions, enlargedDimensions] = fromCell.item.largeBaseSize
+    ? [largeTileDimensions(g), extraLargeTileDimensions(g)]
+    : [[1, 1], largeTileDimensions(g)];
+  // The target dimensions, which toggle between the base and enlarged sizes
   const [toWidth, toHeight] =
-    fromWidth === 1 && fromHeight === 1
-      ? [Math.min(3, Math.max(2, g.columns - 1)), 2]
-      : [1, 1];
+    fromWidth === baseDimensions[0] && fromHeight === baseDimensions[1]
+      ? enlargedDimensions
+      : baseDimensions;
 
   // If we're expanding the tile, we want to create enough new rows at the
   // tile's target position such that every new unit of grid area created during
@@ -319,12 +534,6 @@ export function cycleTileSize(tileId: string, g: Grid): Grid {
     Math.ceil((toWidth * toHeight - fromWidth * fromHeight) / g.columns)
   );
 
-  // This is the grid with the new rows added
-  const gappyGrid: Grid = {
-    ...g,
-    cells: new Array(g.cells.length + newRows * g.columns),
-  };
-
   // The next task is to scan for a spot to place the modified tile. Since we
   // might be creating new rows at the target position, this spot can be shorter
   // than the target height.
@@ -334,8 +543,8 @@ export function cycleTileSize(tileId: string, g: Grid): Grid {
   // To make the tile appear to expand outwards from its center, we're actually
   // scanning for locations to put the *center* of the tile. These numbers are
   // the offsets between the tile's origin and its center.
-  const scanColumnOffset = Math.floor((toWidth - 1) / 2);
-  const scanRowOffset = Math.floor((toHeight - 1) / 2);
+  const scanColumnOffset = Math.floor((toWidth - fromWidth) / 2);
+  const scanRowOffset = Math.floor((toHeight - fromHeight) / 2);
 
   const nextScanLocations = new Set<number>([from]);
   const rows = row(g.cells.length - 1, g) + 1;
@@ -379,17 +588,19 @@ export function cycleTileSize(tileId: string, g: Grid): Grid {
 
   const toRow = row(to, g);
 
-  // Copy tiles from the original grid to the new one, with the new rows
-  // inserted at the target location
-  g.cells.forEach((c, src) => {
-    if (c?.origin && c.item.id !== tileId) {
-      const offset =
-        row(src, g) > toRow + candidateHeight - 1 ? g.columns * newRows : 0;
-      forEachCellInArea(src, areaEnd(src, c.columns, c.rows, g), g, (c, i) => {
-        gappyGrid.cells[i + offset] = c;
-      });
-    }
-  });
+  // This is the grid with the new rows added
+  const gappyGrid = createRows(g, newRows, toRow + candidateHeight);
+
+  // Remove the original tile
+  const fromInGappyGrid =
+    from + (row(from, g) >= toRow + candidateHeight ? g.columns * newRows : 0);
+  const fromEndInGappyGrid = fromInGappyGrid - from + fromEnd;
+  forEachCellInArea(
+    fromInGappyGrid,
+    fromEndInGappyGrid,
+    gappyGrid,
+    (_c, i) => (gappyGrid.cells[i] = undefined)
+  );
 
   // Place the tile in its target position, making a note of the tiles being
   // overwritten
@@ -414,3 +625,79 @@ export function cycleTileSize(tileId: string, g: Grid): Grid {
   // Fill any gaps that remain
   return fillGaps(gappyGrid);
 }
+
+/**
+ * Resizes the grid to a new column width.
+ */
+export function resize(g: Grid, columns: number): Grid {
+  const result: Grid = { columns, cells: [] };
+  const [largeColumns, largeRows] = largeTileDimensions(result);
+
+  // Copy each tile from the old grid to the resized one in the same order
+
+  // The next index in the result grid to copy a tile to
+  let next = 0;
+
+  for (const cell of g.cells) {
+    if (cell?.origin) {
+      const [nextColumns, nextRows] =
+        cell.columns > 1 || cell.rows > 1 ? [largeColumns, largeRows] : [1, 1];
+
+      // If there isn't enough space left on this row, jump to the next row
+      if (columns - column(next, result) < nextColumns)
+        next = columns * (Math.floor(next / columns) + 1);
+      const nextEnd = areaEnd(next, nextColumns, nextRows, result);
+
+      // Expand the cells array as necessary
+      if (result.cells.length <= nextEnd)
+        result.cells.push(...new Array(nextEnd + 1 - result.cells.length));
+
+      // Copy the tile into place
+      forEachCellInArea(next, nextEnd, result, (_c, i) => {
+        result.cells[i] = {
+          item: cell.item,
+          origin: i === next,
+          columns: nextColumns,
+          rows: nextRows,
+        };
+      });
+
+      next = nextEnd + 1;
+    }
+  }
+
+  return fillGaps(result);
+}
+
+/**
+ * Promotes speakers to the first page of the grid.
+ */
+export function promoteSpeakers(g: Grid) {
+  // This is all a bit of a hack right now, because we don't know if the designs
+  // will stick with this approach in the long run
+  // We assume that 4 rows are probably about 1 page
+  const firstPageEnd = g.columns * 4;
+
+  for (let from = firstPageEnd; from < g.cells.length; from++) {
+    const fromCell = g.cells[from];
+    // Don't bother trying to promote enlarged tiles
+    if (
+      fromCell?.item.isSpeaker &&
+      fromCell.columns === 1 &&
+      fromCell.rows === 1
+    ) {
+      // Promote this tile by making 10 attempts to place it on the first page
+      for (let j = 0; j < 10; j++) {
+        const to = Math.floor(Math.random() * firstPageEnd);
+        const toCell = g.cells[to];
+        if (
+          toCell === undefined ||
+          (toCell.columns === 1 && toCell.rows === 1)
+        ) {
+          moveTile(g, from, to);
+          break;
+        }
+      }
+    }
+  }
+}

test/video-grid/model-test.ts

@@ -15,13 +15,15 @@ limitations under the License.
 */
 
 import {
-  appendItems,
+  addItems,
   column,
   cycleTileSize,
   fillGaps,
   forEachCellInArea,
   Grid,
+  resize,
   row,
+  tryMoveTile,
 } from "../../src/video-grid/model";
 import { TileDescriptor } from "../../src/video-grid/TileDescriptor";
 
@@ -31,7 +33,7 @@ import { TileDescriptor } from "../../src/video-grid/TileDescriptor";
 function mkGrid(spec: string): Grid {
   const secondNewline = spec.indexOf("\n", 1);
   const columns = secondNewline === -1 ? spec.length : secondNewline - 1;
-  const cells = spec.match(/[a-z ]/g) ?? [];
+  const cells = spec.match(/[a-z ]/g) ?? ([] as string[]);
   const areas = new Set(cells);
   areas.delete(" "); // Space represents an empty cell, not an area
   const grid: Grid = { columns, cells: new Array(cells.length) };
@@ -169,6 +171,50 @@ dddd
 iegh`
 );
 
+testFillGaps(
+  "keeps a large tile from hanging off the bottom",
+  `
+abcd
+efgh
+    
+ii  
+ii`,
+  `
+abcd
+iigh
+iief`
+);
+
+testFillGaps(
+  "pushes a chain of large tiles upwards",
+  `
+abcd
+e fg
+hh  
+hh  
+ ii 
+ ii`,
+  `
+hhcd
+hhfg
+aiib
+eii`
+);
+
+testFillGaps(
+  "gives up on pushing large tiles upwards when not possible",
+  `
+aabb
+aabb
+cc  
+cc`,
+  `
+aabb
+aabb
+cc  
+cc`
+);
+
 function testCycleTileSize(
   title: string,
   tileId: string,
@@ -227,9 +273,9 @@ dbbe
 fghi
 jk`,
   `
-abhc
-djge
-fik`
+akbc
+djhe
+fig`
 );
 
 testCycleTileSize(
@@ -267,17 +313,160 @@ dde
 ddf`
 );
 
-test("appendItems appends 1×1 tiles", () => {
-  const grid1 = `
+function testAddItems(
+  title: string,
+  items: TileDescriptor[],
+  input: string,
+  output: string
+): void {
+  test(`addItems ${title}`, () => {
+    expect(showGrid(addItems(items, mkGrid(input)))).toBe(output);
+  });
+}
+
+testAddItems(
+  "appends 1×1 tiles",
+  ["e", "f"].map((i) => ({ id: i } as unknown as TileDescriptor)),
+  `
 aab
 aac
-d`;
-  const grid2 = `
+d`,
+  `
 aab
 aac
-def`;
-  const newItems = ["e", "f"].map(
-    (i) => ({ id: i } as unknown as TileDescriptor)
-  );
-  expect(showGrid(appendItems(newItems, mkGrid(grid1)))).toBe(grid2);
-});
+def`
+);
+
+testAddItems(
+  "places one tile near another on request",
+  [{ id: "g", placeNear: "b" } as unknown as TileDescriptor],
+  `
+abc
+def`,
+  `
+abc
+gfe
+d`
+);
+
+testAddItems(
+  "places items with a large base size",
+  [{ id: "g", largeBaseSize: true } as unknown as TileDescriptor],
+  `
+abc
+def`,
+  `
+abc
+ggf
+gge
+d`
+);
+
+function testTryMoveTile(
+  title: string,
+  from: number,
+  to: number,
+  input: string,
+  output: string
+): void {
+  test(`tryMoveTile ${title}`, () => {
+    expect(showGrid(tryMoveTile(mkGrid(input), from, to))).toBe(output);
+  });
+}
+
+testTryMoveTile(
+  "refuses to move a tile too far to the left",
+  1,
+  -1,
+  `
+abc`,
+  `
+abc`
+);
+
+testTryMoveTile(
+  "refuses to move a tile too far to the right",
+  1,
+  3,
+  `
+abc`,
+  `
+abc`
+);
+
+testTryMoveTile(
+  "moves a large tile to an unoccupied space",
+  3,
+  1,
+  `
+a b
+ccd
+cce`,
+  `
+acc
+bcc
+d e`
+);
+
+testTryMoveTile(
+  "refuses to move a large tile to an occupied space",
+  3,
+  1,
+  `
+abb
+ccd
+cce`,
+  `
+abb
+ccd
+cce`
+);
+
+function testResize(
+  title: string,
+  columns: number,
+  input: string,
+  output: string
+): void {
+  test(`resize ${title}`, () => {
+    expect(showGrid(resize(mkGrid(input), columns))).toBe(output);
+  });
+}
+
+testResize(
+  "contracts the grid",
+  2,
+  `
+abbb
+cbbb
+ddde
+dddf
+gh`,
+  `
+af
+bb
+bb
+ch
+dd
+dd
+eg`
+);
+
+testResize(
+  "expands the grid",
+  4,
+  `
+af
+bb
+bb
+ch
+dd
+dd
+eg`,
+  `
+bbbc
+bbbf
+addd
+hddd
+ge`
+);

yarn.lock

@@ -1828,10 +1828,10 @@
   resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.3.1.tgz#b50a781709c81e10701004214340f25475a171a0"
   integrity sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw==
 
-"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.9":
-  version "0.1.0-alpha.9"
-  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.9.tgz#00bc266781502641a661858a5a521dd4d95275fc"
-  integrity sha512-g5cjpFwA9h0CbEGoAqNVI2QcyDsbI8FHoLo9+OXWHIezEKITsSv78mc5ilIwN+2YpmVlH0KNeQWTHw4vi0BMnw==
+"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.10":
+  version "0.1.0-alpha.11"
+  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.11.tgz#24d705318c3159ef7dbe43bca464ac2bdd11e45d"
+  integrity sha512-HD3rskPkqrUUSaKzGLg97k/bN+OZrkcX7ODB/pNBs/jqq+/A0wDKqsszJotzFwsQcDPpWn78BmMyvBo4tLxKjw==
 
 "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz":
   version "3.2.14"
@@ -10557,12 +10557,12 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1":
-  version "26.0.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1"
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f884c78579c336a03bc20ff8f4e92c46582822b6":
+  version "26.1.0"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f884c78579c336a03bc20ff8f4e92c46582822b6"
   dependencies:
     "@babel/runtime" "^7.12.5"
-    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9"
+    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.10"
     another-json "^0.2.0"
     bs58 "^5.0.0"
     content-type "^1.0.4"