[__ungleich_matrix] wire for worker mode

type/__ungleich_matrix/files/nginx-upstream-config.sh

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+set -e
+
+generate_upstream_block () {
+	name=$1
+	workers=$2
+	balancing_method=$3
+
+	if [ -n "$workers" ]; then
+		echo "upstream $name {"
+
+		for server in $workers; do
+			echo "	server $server;"
+		done
+
+		if [ -n "$balancing_method" ]; then
+			echo "$balancing_method;"
+		fi
+
+		echo "}"
+	fi
+}
+
+# Inbound federation requests go to the same worker (based on IP).
+generate_upstream_block inbound_federation_workers "$FEDERATION_WORKERS" ip_hash
+
+# For /sync and /initialSync requests it will be more efficient if all requests
+# from a particular user are routed to a single instance. Extracting a user ID
+# from the access token or Authorization header is a pain, so we just group by
+# IP address: a specific user will use one address per client, but should not
+# have too many clients...
+generate_upstream_block sync_workers "$SYNC_WORKERS" ip_hash
+
+# Client, event seding and outbound federation are round-robin.
+generate_upstream_block federation_workers "$FEDERATION_WORKERS"
+generate_upstream_block client_workers "$CLIENT_WORKERS"
+generate_upstream_block event_sending_workers "$EVENT_SENDING_WORKERS"

type/__ungleich_matrix/files/nginx-vhost-config.sh

@@ -0,0 +1,139 @@
+#!/bin/sh
+
+set -e
+
+sync_endpoints=$(tr -d '\n' << EOF
+(^/_matrix/client/(v2_alpha|r0)/sync$
+|^/_matrix/client/(api/v1|v2_alpha|r0)/events$
+|^/_matrix/client/(api/v1|r0)/initialSync$
+|^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$)
+EOF
+)
+
+federation_endpoints=$(tr -d '\n' << EOF
+(^/_matrix/federation/v1/event/
+|^/_matrix/federation/v1/state/
+|^/_matrix/federation/v1/state_ids/
+|^/_matrix/federation/v1/backfill/
+|^/_matrix/federation/v1/get_missing_events/
+|^/_matrix/federation/v1/publicRooms
+|^/_matrix/federation/v1/query/
+|^/_matrix/federation/v1/make_join/
+|^/_matrix/federation/v1/make_leave/
+|^/_matrix/federation/v1/send_join/
+|^/_matrix/federation/v2/send_join/
+|^/_matrix/federation/v1/send_leave/
+|^/_matrix/federation/v2/send_leave/
+|^/_matrix/federation/v1/invite/
+|^/_matrix/federation/v2/invite/
+|^/_matrix/federation/v1/query_auth/
+|^/_matrix/federation/v1/event_auth/
+|^/_matrix/federation/v1/exchange_third_party_invite/
+|^/_matrix/federation/v1/user/devices/
+|^/_matrix/federation/v1/get_groups_publicised$
+|^/_matrix/key/v2/query)
+EOF
+)
+
+inbound_federation_endpoint="^/_matrix/federation/v1/send/"
+
+event_sending_endpoints=$(tr -d '\n' << EOF
+(^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$
+|^/_matrix/client/(api/v1|r0|unstable)/join/
+|^/_matrix/client/(api/v1|r0|unstable)/profile/)
+EOF
+)
+
+client_endpoints=$(tr -d '\n' << EOF
+(^/_matrix/client/(api/v1|r0|unstable)/publicRooms$
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$
+|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$
+|^/_matrix/client/(api/v1|r0|unstable)/account/3pid$
+|^/_matrix/client/(api/v1|r0|unstable)/devices$
+|^/_matrix/client/(api/v1|r0|unstable)/keys/query$
+|^/_matrix/client/(api/v1|r0|unstable)/keys/changes$
+|^/_matrix/client/versions$
+|^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$
+|^/_matrix/client/(api/v1|r0|unstable)/joined_groups$
+|^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$
+|^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/)
+EOF
+)
+
+cat << EOF
+# Deny access to root.
+deny all;
+
+location ~ /_matrix|/_synapse {
+	# Allow anyone to reach synapse.
+	allow all;
+
+	# Allow uploading large files.
+	client_max_body_size ${MAX_UPLOAD_SIZE:?};
+
+	# Proxy configuration.
+	proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+	proxy_set_header X-Forwarded-Proto \$scheme;
+	proxy_set_header Host \$http_host;
+	proxy_set_header X-Real-IP \$remote_addr;
+	proxy_pass http://localhost:8008;
+
+	location ~ /_synapse/metrics {
+		# service-monitoring.p6 (monitoring LAN).
+		allow $PROMETHEUS_SOURCE_ADDRESS;
+		deny all;
+
+		proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
+		proxy_set_header X-Forwarded-Proto \$scheme;
+		proxy_set_header Host \$http_host;
+		proxy_set_header X-Real-IP \$remote_addr;
+		proxy_pass http://localhost:8008;
+	}
+EOF
+
+if [ -n "$FEDERATION_WORKERS" ]; then
+	cat << EOF
+	# Inbound federation.
+	location ~ $inbound_federation_endpoint {
+		proxy_pass http://inbound_federation_workers;
+	}
+
+	# Anything else federation.
+	location ~ $federation_endpoints {
+		proxy_pass http://federation_workers;
+	}
+EOF
+fi
+
+if [ -n "$SYNC_WORKERS" ]; then
+	cat << EOF
+	# Syncing
+	location ~ $sync_endpoints {
+		proxy_pass http://sync_workers;
+	}
+EOF
+fi
+
+if [ -n "$EVENT_SENDING_WORKERS" ]; then
+	cat << EOF
+	location ~ $event_sending_endpoints {
+
+		proxy_pass http://event_sending_workers;
+	}
+EOF
+fi
+
+if [ -n "$CLIENT_WORKERS" ]; then
+	cat << EOF
+	location ~ $client_endpoints {
+		proxy_pass http://client_workers;
+	}
+EOF
+fi
+
+echo "}"

type/__ungleich_matrix/man.rst

@@ -46,10 +46,25 @@ element-address
 element-version
   Version of the Element client to be deployed.
 
+sync-worker
+  Address of worker processing sync requests. Can be
+  specified multiple times.
+
+federation-worker
+  Address of worker processing federation requests.
+  Can be specified multiple times.
+
+client-worker
+  Address of worker processing client requests. Can
+  be specified multiple times.
+
+event-sending-worker
+  Address of worker processing event sending requests. Can be specified multiple times.
+
 BOOLEAN PARAMETERS
 ------------------
-None.
+synapse-worker-mode
-
+  Enables synapse's worker-mode.
 
 EXAMPLES
 --------

type/__ungleich_matrix/manifest

@@ -15,13 +15,13 @@ fi
 # Type-level flags. Feel free to change them.
 
 # Nginx and synapse maximum size for uploaded files.
-MAX_UPLOAD_SIZE=100M
+export MAX_UPLOAD_SIZE=100M
 
 # Default domain for Jitsi
 JITSI_DOMAIN=talk.ungleich.ch
 
 # (Source) address used by prometheus to fetch synapse metrics.
-PROMETHEUS_SOURCE_ADDRESS=2a0a:e5c0:2:12:0:f0ff:fea9:c461
+export PROMETHEUS_SOURCE_ADDRESS=2a0a:e5c0:2:12:0:f0ff:fea9:c461
 
 # ungleich's privacy policy - displayed in element web client.
 PRIVACY_POLICY_URL=https://redmine.ungleich.ch/projects/open-infrastructure/wiki/Security_and_Privacy_Policy
@@ -51,10 +51,36 @@ synapse_smtp_password=$(cat "$__object/parameter/synapse-smtp-password")
 if [ -f "$__object/parameter/synapse-extra-parameters" ]; then
 	synapse_extra_parameters=$(cat "$__object/parameter/synapse-extra-parameters")
 fi
+
 if [ -f "$__object/parameter/element-extra-parameters" ]; then
 	element_extra_parameters=$(cat "$__object/parameter/element-extra-parameters")
 fi
 
+synapse_worker_mode=
+if [ -f "$__object/parameter/synapse-worker-mode" ]; then
+	synapse_worker_mode='--worker-mode'
+fi
+
+if [ -f "$__object/parameter/sync-worker" ]; then
+	SYNC_WORKERS=$(cat "$__object/parameter/sync-worker")
+	export SYNC_WORKERS
+fi
+
+if [ -f "$__object/parameter/federation-worker" ]; then
+	FEDERATION_WORKERS=$(cat "$__object/parameter/federation-worker")
+	export FEDERATION_WORKERS
+fi
+
+if [ -f "$__object/parameter/client-worker" ]; then
+	CLIENT_WORKERS=$(cat "$__object/parameter/client-worker")
+	export CLIENT_WORKERS
+fi
+
+if [ -f "$__object/parameter/event-sending-worker" ]; then
+	EVENT_SENDING_WORKERS=$(cat "$__object/parameter/event-sending-worker")
+	export EVENT_SENDING_WORKERS
+fi
+
 # Generic configuration - shared with all ungleich Matrix deployments.
 synapse_base_url="https://$synapse_domain"
 
@@ -91,8 +117,19 @@ require="__postgres_role/$postgres_user" __postgres_database $postgres_user \
 	--template template0
 
 # Install & configure Synapse (matrix homeserver).
+synapse_reqs=
+if [ -n "$synapse_worker_mode" ]; then
+	__package redis
+	__package python3-hiredis
+	__package python3-pip
+	require="__package/python3-pip" __package_pip txredisapi
+
+	synapse_reqs="__package/python3-hiredis __package_pip/txredisapi \
+		__package/redis"
+fi
+
 # shellcheck disable=SC2086
-__matrix_synapse \
+require="$synapse_reqs" __matrix_synapse \
 	--server-name "$matrix_domain" \
 	--base-url "$synapse_base_url" \
 	--max-upload-size "$MAX_UPLOAD_SIZE" \
@@ -108,50 +145,25 @@ __matrix_synapse \
 	--smtp-use-starttls \
 	--smtp-user "$synapse_smtp_user" \
 	--smtp-pass "$synapse_smtp_password" \
+	$synapse_worker_mode \
 	$synapse_extra_parameters
 
 # Install and configure NGINX web server/proxy.
 __package nginx
 
-synapse_nginx_config="$(cat << EOF
+require="__package/nginx" __file /etc/nginx/sites-enabled/synapse-upstreams \
-# Deny access to root.
+	--mode 0644 \
-deny all;
+	--onchange "service nginx reload" \
-
+	--source - << EOF
-location ~ /_matrix|/_synapse {
+$("$__type"/files/nginx-upstream-config.sh)
-    # Allow anyone to reach synapse.
-    allow all;
-
-    # Allow uploading large files.
-    client_max_body_size ${MAX_UPLOAD_SIZE:?};
-
-    # Proxy configuration.
-    proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-    proxy_set_header X-Forwarded-Proto \$scheme;
-    proxy_set_header Host \$http_host;
-    proxy_set_header X-Real-IP \$remote_addr;
-    proxy_pass http://localhost:8008;
-
-    location ~ /_synapse/metrics {
-        # service-monitoring.p6 (monitoring LAN).
-        allow $PROMETHEUS_SOURCE_ADDRESS;
-        deny all;
-
-        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
-        proxy_set_header X-Forwarded-Proto \$scheme;
-        proxy_set_header Host \$http_host;
-        proxy_set_header X-Real-IP \$remote_addr;
-        proxy_pass http://localhost:8008;
-      }
-  }
 EOF
-)"
 
-require="__matrix_synapse __package/nginx" \
+require="__package/nginx __file/etc/nginx/sites-enabled/synapse-upstreams" \
 	__ungleich_nginx_static_site "$synapse_domain" \
 	--owner "$www_directory_owner" \
 	--listen '443 [::]:443' \
 	--base_directory "$nginx_basedir" \
-	--locationopt "$synapse_nginx_config"
+	--locationopt "$("$__type"/files/nginx-vhost-config.sh)"
 
 # Delegate Matrix federation to port 443 & configure server discovery from
 # clients if matrix_domain is element_domain (= both are handled by this

type/__ungleich_matrix/parameter/boolean

@@ -0,0 +1 @@
+synapse-worker-mode

type/__ungleich_matrix/parameter/optional_multiple

@@ -0,0 +1,4 @@
+sync-worker
+federation-worker
+client-worker
+event-sending-worker