From d58eef93513f643f4979238d2e669518bb06a8ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 19 Feb 2021 11:27:16 +0100 Subject: [PATCH] [__ungleich_matrix] wire for worker mode --- .../files/nginx-upstream-config.sh | 38 +++++ .../files/nginx-vhost-config.sh | 139 ++++++++++++++++++ type/__ungleich_matrix/man.rst | 19 ++- type/__ungleich_matrix/manifest | 84 ++++++----- type/__ungleich_matrix/parameter/boolean | 1 + .../parameter/optional_multiple | 4 + 6 files changed, 247 insertions(+), 38 deletions(-) create mode 100755 type/__ungleich_matrix/files/nginx-upstream-config.sh create mode 100755 type/__ungleich_matrix/files/nginx-vhost-config.sh create mode 100644 type/__ungleich_matrix/parameter/boolean create mode 100644 type/__ungleich_matrix/parameter/optional_multiple diff --git a/type/__ungleich_matrix/files/nginx-upstream-config.sh b/type/__ungleich_matrix/files/nginx-upstream-config.sh new file mode 100755 index 0000000..63bb4d3 --- /dev/null +++ b/type/__ungleich_matrix/files/nginx-upstream-config.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +set -e + +generate_upstream_block () { + name=$1 + workers=$2 + balancing_method=$3 + + if [ -n "$workers" ]; then + echo "upstream $name {" + + for server in $workers; do + echo " server $server;" + done + + if [ -n "$balancing_method" ]; then + echo "$balancing_method;" + fi + + echo "}" + fi +} + +# Inbound federation requests go to the same worker (based on IP). +generate_upstream_block inbound_federation_workers "$FEDERATION_WORKERS" ip_hash + +# For /sync and /initialSync requests it will be more efficient if all requests +# from a particular user are routed to a single instance. Extracting a user ID +# from the access token or Authorization header is a pain, so we just group by +# IP address: a specific user will use one address per client, but should not +# have too many clients... +generate_upstream_block sync_workers "$SYNC_WORKERS" ip_hash + +# Client, event seding and outbound federation are round-robin. +generate_upstream_block federation_workers "$FEDERATION_WORKERS" +generate_upstream_block client_workers "$CLIENT_WORKERS" +generate_upstream_block event_sending_workers "$EVENT_SENDING_WORKERS" diff --git a/type/__ungleich_matrix/files/nginx-vhost-config.sh b/type/__ungleich_matrix/files/nginx-vhost-config.sh new file mode 100755 index 0000000..f28f8db --- /dev/null +++ b/type/__ungleich_matrix/files/nginx-vhost-config.sh @@ -0,0 +1,139 @@ +#!/bin/sh + +set -e + +sync_endpoints=$(tr -d '\n' << EOF +(^/_matrix/client/(v2_alpha|r0)/sync$ +|^/_matrix/client/(api/v1|v2_alpha|r0)/events$ +|^/_matrix/client/(api/v1|r0)/initialSync$ +|^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$) +EOF +) + +federation_endpoints=$(tr -d '\n' << EOF +(^/_matrix/federation/v1/event/ +|^/_matrix/federation/v1/state/ +|^/_matrix/federation/v1/state_ids/ +|^/_matrix/federation/v1/backfill/ +|^/_matrix/federation/v1/get_missing_events/ +|^/_matrix/federation/v1/publicRooms +|^/_matrix/federation/v1/query/ +|^/_matrix/federation/v1/make_join/ +|^/_matrix/federation/v1/make_leave/ +|^/_matrix/federation/v1/send_join/ +|^/_matrix/federation/v2/send_join/ +|^/_matrix/federation/v1/send_leave/ +|^/_matrix/federation/v2/send_leave/ +|^/_matrix/federation/v1/invite/ +|^/_matrix/federation/v2/invite/ +|^/_matrix/federation/v1/query_auth/ +|^/_matrix/federation/v1/event_auth/ +|^/_matrix/federation/v1/exchange_third_party_invite/ +|^/_matrix/federation/v1/user/devices/ +|^/_matrix/federation/v1/get_groups_publicised$ +|^/_matrix/key/v2/query) +EOF +) + +inbound_federation_endpoint="^/_matrix/federation/v1/send/" + +event_sending_endpoints=$(tr -d '\n' << EOF +(^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/redact +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/ +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$ +|^/_matrix/client/(api/v1|r0|unstable)/join/ +|^/_matrix/client/(api/v1|r0|unstable)/profile/) +EOF +) + +client_endpoints=$(tr -d '\n' << EOF +(^/_matrix/client/(api/v1|r0|unstable)/publicRooms$ +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$ +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$ +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$ +|^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$ +|^/_matrix/client/(api/v1|r0|unstable)/account/3pid$ +|^/_matrix/client/(api/v1|r0|unstable)/devices$ +|^/_matrix/client/(api/v1|r0|unstable)/keys/query$ +|^/_matrix/client/(api/v1|r0|unstable)/keys/changes$ +|^/_matrix/client/versions$ +|^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$ +|^/_matrix/client/(api/v1|r0|unstable)/joined_groups$ +|^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$ +|^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/) +EOF +) + +cat << EOF +# Deny access to root. +deny all; + +location ~ /_matrix|/_synapse { + # Allow anyone to reach synapse. + allow all; + + # Allow uploading large files. + client_max_body_size ${MAX_UPLOAD_SIZE:?}; + + # Proxy configuration. + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header Host \$http_host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_pass http://localhost:8008; + + location ~ /_synapse/metrics { + # service-monitoring.p6 (monitoring LAN). + allow $PROMETHEUS_SOURCE_ADDRESS; + deny all; + + proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto \$scheme; + proxy_set_header Host \$http_host; + proxy_set_header X-Real-IP \$remote_addr; + proxy_pass http://localhost:8008; + } +EOF + +if [ -n "$FEDERATION_WORKERS" ]; then + cat << EOF + # Inbound federation. + location ~ $inbound_federation_endpoint { + proxy_pass http://inbound_federation_workers; + } + + # Anything else federation. + location ~ $federation_endpoints { + proxy_pass http://federation_workers; + } +EOF +fi + +if [ -n "$SYNC_WORKERS" ]; then + cat << EOF + # Syncing + location ~ $sync_endpoints { + proxy_pass http://sync_workers; + } +EOF +fi + +if [ -n "$EVENT_SENDING_WORKERS" ]; then + cat << EOF + location ~ $event_sending_endpoints { + + proxy_pass http://event_sending_workers; + } +EOF +fi + +if [ -n "$CLIENT_WORKERS" ]; then + cat << EOF + location ~ $client_endpoints { + proxy_pass http://client_workers; + } +EOF +fi + +echo "}" diff --git a/type/__ungleich_matrix/man.rst b/type/__ungleich_matrix/man.rst index b8583e4..e2201df 100644 --- a/type/__ungleich_matrix/man.rst +++ b/type/__ungleich_matrix/man.rst @@ -46,10 +46,25 @@ element-address element-version Version of the Element client to be deployed. +sync-worker + Address of worker processing sync requests. Can be + specified multiple times. + +federation-worker + Address of worker processing federation requests. + Can be specified multiple times. + +client-worker + Address of worker processing client requests. Can + be specified multiple times. + +event-sending-worker + Address of worker processing event sending requests. Can be specified multiple times. + BOOLEAN PARAMETERS ------------------ -None. - +synapse-worker-mode + Enables synapse's worker-mode. EXAMPLES -------- diff --git a/type/__ungleich_matrix/manifest b/type/__ungleich_matrix/manifest index b132404..7a93dfa 100755 --- a/type/__ungleich_matrix/manifest +++ b/type/__ungleich_matrix/manifest @@ -15,13 +15,13 @@ fi # Type-level flags. Feel free to change them. # Nginx and synapse maximum size for uploaded files. -MAX_UPLOAD_SIZE=100M +export MAX_UPLOAD_SIZE=100M # Default domain for Jitsi JITSI_DOMAIN=talk.ungleich.ch # (Source) address used by prometheus to fetch synapse metrics. -PROMETHEUS_SOURCE_ADDRESS=2a0a:e5c0:2:12:0:f0ff:fea9:c461 +export PROMETHEUS_SOURCE_ADDRESS=2a0a:e5c0:2:12:0:f0ff:fea9:c461 # ungleich's privacy policy - displayed in element web client. PRIVACY_POLICY_URL=https://redmine.ungleich.ch/projects/open-infrastructure/wiki/Security_and_Privacy_Policy @@ -51,10 +51,36 @@ synapse_smtp_password=$(cat "$__object/parameter/synapse-smtp-password") if [ -f "$__object/parameter/synapse-extra-parameters" ]; then synapse_extra_parameters=$(cat "$__object/parameter/synapse-extra-parameters") fi + if [ -f "$__object/parameter/element-extra-parameters" ]; then element_extra_parameters=$(cat "$__object/parameter/element-extra-parameters") fi +synapse_worker_mode= +if [ -f "$__object/parameter/synapse-worker-mode" ]; then + synapse_worker_mode='--worker-mode' +fi + +if [ -f "$__object/parameter/sync-worker" ]; then + SYNC_WORKERS=$(cat "$__object/parameter/sync-worker") + export SYNC_WORKERS +fi + +if [ -f "$__object/parameter/federation-worker" ]; then + FEDERATION_WORKERS=$(cat "$__object/parameter/federation-worker") + export FEDERATION_WORKERS +fi + +if [ -f "$__object/parameter/client-worker" ]; then + CLIENT_WORKERS=$(cat "$__object/parameter/client-worker") + export CLIENT_WORKERS +fi + +if [ -f "$__object/parameter/event-sending-worker" ]; then + EVENT_SENDING_WORKERS=$(cat "$__object/parameter/event-sending-worker") + export EVENT_SENDING_WORKERS +fi + # Generic configuration - shared with all ungleich Matrix deployments. synapse_base_url="https://$synapse_domain" @@ -91,8 +117,19 @@ require="__postgres_role/$postgres_user" __postgres_database $postgres_user \ --template template0 # Install & configure Synapse (matrix homeserver). +synapse_reqs= +if [ -n "$synapse_worker_mode" ]; then + __package redis + __package python3-hiredis + __package python3-pip + require="__package/python3-pip" __package_pip txredisapi + + synapse_reqs="__package/python3-hiredis __package_pip/txredisapi \ + __package/redis" +fi + # shellcheck disable=SC2086 -__matrix_synapse \ +require="$synapse_reqs" __matrix_synapse \ --server-name "$matrix_domain" \ --base-url "$synapse_base_url" \ --max-upload-size "$MAX_UPLOAD_SIZE" \ @@ -108,50 +145,25 @@ __matrix_synapse \ --smtp-use-starttls \ --smtp-user "$synapse_smtp_user" \ --smtp-pass "$synapse_smtp_password" \ + $synapse_worker_mode \ $synapse_extra_parameters # Install and configure NGINX web server/proxy. __package nginx -synapse_nginx_config="$(cat << EOF -# Deny access to root. -deny all; - -location ~ /_matrix|/_synapse { - # Allow anyone to reach synapse. - allow all; - - # Allow uploading large files. - client_max_body_size ${MAX_UPLOAD_SIZE:?}; - - # Proxy configuration. - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto \$scheme; - proxy_set_header Host \$http_host; - proxy_set_header X-Real-IP \$remote_addr; - proxy_pass http://localhost:8008; - - location ~ /_synapse/metrics { - # service-monitoring.p6 (monitoring LAN). - allow $PROMETHEUS_SOURCE_ADDRESS; - deny all; - - proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto \$scheme; - proxy_set_header Host \$http_host; - proxy_set_header X-Real-IP \$remote_addr; - proxy_pass http://localhost:8008; - } - } +require="__package/nginx" __file /etc/nginx/sites-enabled/synapse-upstreams \ + --mode 0644 \ + --onchange "service nginx reload" \ + --source - << EOF +$("$__type"/files/nginx-upstream-config.sh) EOF -)" -require="__matrix_synapse __package/nginx" \ +require="__package/nginx __file/etc/nginx/sites-enabled/synapse-upstreams" \ __ungleich_nginx_static_site "$synapse_domain" \ --owner "$www_directory_owner" \ --listen '443 [::]:443' \ --base_directory "$nginx_basedir" \ - --locationopt "$synapse_nginx_config" + --locationopt "$("$__type"/files/nginx-vhost-config.sh)" # Delegate Matrix federation to port 443 & configure server discovery from # clients if matrix_domain is element_domain (= both are handled by this diff --git a/type/__ungleich_matrix/parameter/boolean b/type/__ungleich_matrix/parameter/boolean new file mode 100644 index 0000000..a30f811 --- /dev/null +++ b/type/__ungleich_matrix/parameter/boolean @@ -0,0 +1 @@ +synapse-worker-mode diff --git a/type/__ungleich_matrix/parameter/optional_multiple b/type/__ungleich_matrix/parameter/optional_multiple new file mode 100644 index 0000000..c4e4192 --- /dev/null +++ b/type/__ungleich_matrix/parameter/optional_multiple @@ -0,0 +1,4 @@ +sync-worker +federation-worker +client-worker +event-sending-worker