From 9c56ac7063234ad59ca5bf55b3ca4d1b7cf0cc6f Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Fri, 30 Jul 2021 20:14:40 +0200
Subject: [PATCH 01/13] Add trusted domains to the nextcloud environment

---
 apps/nextcloud/templates/deployment.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/apps/nextcloud/templates/deployment.yaml b/apps/nextcloud/templates/deployment.yaml
index fde2e64..5c23752 100644
--- a/apps/nextcloud/templates/deployment.yaml
+++ b/apps/nextcloud/templates/deployment.yaml
@@ -91,6 +91,8 @@ spec:
                 secretKeyRef:
                   name: {{ tpl .Values.identifier . }}-postgres-config
                   key: POSTGRES_DB
+            - name: NEXTCLOUD_TRUSTED_DOMAINS
+              value: "{{ tpl .Values.fqdn . }}"
             - name: NEXTCLOUD_ADMIN_USER
               valueFrom:
                 secretKeyRef:

From 3fece5e44797a5973782805e170c03c2734715b9 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sat, 31 Jul 2021 09:35:47 +0200
Subject: [PATCH 02/13] [nextcloud] upgrade to 21.0.3

---
 apps/nextcloud/Chart.yaml                | 2 +-
 apps/nextcloud/templates/deployment.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/nextcloud/Chart.yaml b/apps/nextcloud/Chart.yaml
index 84c101b..ccb5282 100644
--- a/apps/nextcloud/Chart.yaml
+++ b/apps/nextcloud/Chart.yaml
@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "20.0.11"
+appVersion: "21.0.3"
diff --git a/apps/nextcloud/templates/deployment.yaml b/apps/nextcloud/templates/deployment.yaml
index 5c23752..41a18ca 100644
--- a/apps/nextcloud/templates/deployment.yaml
+++ b/apps/nextcloud/templates/deployment.yaml
@@ -82,7 +82,7 @@ spec:
             - name: nextcloud-data
               mountPath: "/var/www/html"
         - name: nextcloud
-          image: nextcloud:20.0.11-fpm
+          image: nextcloud:{{ .Chart.AppVersion }}-fpm
           ports:
             - containerPort: 9000
           env:

From e6c3ebef7697ea5c979153eec6797a811aa58ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sat, 31 Jul 2021 10:21:21 +0200
Subject: [PATCH 03/13] [rook] mention workaround for missing provisioners

---
 rook/README.md | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/rook/README.md b/rook/README.md
index 58d0016..627631b 100644
--- a/rook/README.md
+++ b/rook/README.md
@@ -123,3 +123,12 @@ Especially these:
 ## Other flux related problems
 
 * The host is not cleared / old /var/lib/rook is persisting
+
+## Troubleshooting: PVC stuck pending, no csi-{cephfs,rbd}provisioner-plugin pod in rook-ceph namespace
+
+2021-07-31: it seems that the provisioner plugin tend to silently die.
+Restarting the `rook-ceph-operator` deployment will get them back up:
+
+```
+kubectl rollout restart deployment/rook-ceph-operator -n rook-ceph
+```

From 926a552d688aea06a0ef1107410f4dae0be7260c Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sat, 31 Jul 2021 11:07:30 +0200
Subject: [PATCH 04/13] ++service test

---
 tests/service-without-endpoints.yaml | 10 ++++++++++
 1 file changed, 10 insertions(+)
 create mode 100644 tests/service-without-endpoints.yaml

diff --git a/tests/service-without-endpoints.yaml b/tests/service-without-endpoints.yaml
new file mode 100644
index 0000000..1832176
--- /dev/null
+++ b/tests/service-without-endpoints.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: blank-service
+spec:
+  selector:
+    app: something-that-comes-later
+  ports:
+    - protocol: TCP
+      port: 80

From bcf64117d1e78646f38d441b48165790b7a0f240 Mon Sep 17 00:00:00 2001
From: Nico Schottelius <nico@nico-notebook.schottelius.org>
Date: Sat, 31 Jul 2021 13:05:19 +0200
Subject: [PATCH 05/13] [nextcloud] add probes && switch to alpine

---
 apps/nextcloud/templates/deployment.yaml | 35 +++++++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/apps/nextcloud/templates/deployment.yaml b/apps/nextcloud/templates/deployment.yaml
index 41a18ca..32d2318 100644
--- a/apps/nextcloud/templates/deployment.yaml
+++ b/apps/nextcloud/templates/deployment.yaml
@@ -81,10 +81,43 @@ spec:
               mountPath: "/etc/letsencrypt"
             - name: nextcloud-data
               mountPath: "/var/www/html"
+          # Is it ready to work?
+          readinessProbe:
+            tcpSocket:
+              port: 443
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          # Is it still working?
+          livenessProbe:
+            tcpSocket:
+              port: 443
+            initialDelaySeconds: 15
+            periodSeconds: 20
         - name: nextcloud
-          image: nextcloud:{{ .Chart.AppVersion }}-fpm
+          image: nextcloud:{{ .Chart.AppVersion }}-fpm-alpine
+          # Wait for 10 minutes to get ready
+          startupProbe:
+            httpGet:
+              path: /ocs/v2.php/apps/serverinfo/api/v1/info
+              port: fpm
+            failureThreshold: 20
+            periodSeconds: 30
+          # Dead if failing for 1 minute
+          livenessProbe:
+            httpGet:
+              path: /ocs/v2.php/apps/serverinfo/api/v1/info
+              port: fpm
+            failureThreshold: 6
+            periodSeconds: 10
+          readinessProbe:
+            httpGet:
+              path: /ocs/v2.php/apps/serverinfo/api/v1/info
+              port: fpm
+            failureThreshold: 3
+            periodSeconds: 30
           ports:
             - containerPort: 9000
+              name: fpm
           env:
             - name: POSTGRES_DB
               valueFrom:

From c9b64f4faf4db4d4b5322cfd407b9a7a870bc299 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sat, 31 Jul 2021 16:04:27 +0200
Subject: [PATCH 06/13] Add minimal openldap Chart

---
 apps/fnux-playground/openldap/.helmignore     | 23 +++++++
 apps/fnux-playground/openldap/Chart.yaml      | 24 +++++++
 .../openldap/templates/deployment.yaml        | 65 +++++++++++++++++++
 .../openldap/templates/pvc.yaml               | 11 ++++
 .../openldap/templates/service.yaml           | 13 ++++
 apps/fnux-playground/openldap/values.yaml     |  7 ++
 6 files changed, 143 insertions(+)
 create mode 100644 apps/fnux-playground/openldap/.helmignore
 create mode 100644 apps/fnux-playground/openldap/Chart.yaml
 create mode 100644 apps/fnux-playground/openldap/templates/deployment.yaml
 create mode 100644 apps/fnux-playground/openldap/templates/pvc.yaml
 create mode 100644 apps/fnux-playground/openldap/templates/service.yaml
 create mode 100644 apps/fnux-playground/openldap/values.yaml

diff --git a/apps/fnux-playground/openldap/.helmignore b/apps/fnux-playground/openldap/.helmignore
new file mode 100644
index 0000000..0e8a0eb
--- /dev/null
+++ b/apps/fnux-playground/openldap/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/apps/fnux-playground/openldap/Chart.yaml b/apps/fnux-playground/openldap/Chart.yaml
new file mode 100644
index 0000000..1b45324
--- /dev/null
+++ b/apps/fnux-playground/openldap/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: openldap
+description: OpenLDAP server
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "1.5.0"
diff --git a/apps/fnux-playground/openldap/templates/deployment.yaml b/apps/fnux-playground/openldap/templates/deployment.yaml
new file mode 100644
index 0000000..9668566
--- /dev/null
+++ b/apps/fnux-playground/openldap/templates/deployment.yaml
@@ -0,0 +1,65 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ tpl .Values.identifier . }}"
+  labels:
+    app: openldap
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: "{{ tpl .Values.identifier . }}-openldap"
+  template:
+    metadata:
+      labels:
+        app: "{{ tpl .Values.identifier . }}-openldap"
+    spec:
+      containers:
+        - name: "openldap"
+          image: "osixia/openldap:{{ .Chart.AppVersion }}"
+          args: ["--loglevel", "info"]
+          ports:
+            - name: ldap
+              containerPort: 389
+              protocol: TCP
+            - name: ldaps
+              containerPort: 636
+              protocol: TCP
+          livenessProbe:
+            tcpSocket:
+              port: 389
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          readinessProbe:
+            tcpSocket:
+              port: 389
+            initialDelaySeconds: 10
+            periodSeconds: 10
+          env:
+            - name: HOSTNAME
+              value: "{{ tpl .Values.fqdn . }}"
+            - name: LDAP_LOG_LEVEL
+              value: "{{ tpl .Values.ldapLogLevel . }}"
+            - name: LDAP_ORGANISATION
+              value: "{{ tpl .Values.ldapOrganisation . }}"
+            - name: LDAP_DOMAIN
+              value: "{{ tpl .Values.ldapDomain . }}"
+            - name: LDAP_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ tpl .Values.identifier . }}-openldap"
+                  key: LDAP_ADMIN_PASSWORD
+          volumeMounts:
+            - name: "{{ tpl .Values.identifier . }}-openldap-data"
+              mountPath: "/etc/ldap/slapd.d"
+              subPath: configuration
+            - name: "{{ tpl .Values.identifier . }}-openldap-data"
+              mountPath: "/var/lib/ldap"
+              subPath: database
+            - name: "{{ tpl .Values.identifier . }}-openldap-data"
+              mountPath: /container/service/slapd/assets/certs
+              subPath: certs
+      volumes:
+        - name: "{{ tpl .Values.identifier . }}-openldap-data"
+          persistentVolumeClaim:
+            claimName: "{{ tpl .Values.identifier . }}-openldap-data"
diff --git a/apps/fnux-playground/openldap/templates/pvc.yaml b/apps/fnux-playground/openldap/templates/pvc.yaml
new file mode 100644
index 0000000..66d55ad
--- /dev/null
+++ b/apps/fnux-playground/openldap/templates/pvc.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "{{ tpl .Values.identifier . }}-openldap-data"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: rook-cephfs
diff --git a/apps/fnux-playground/openldap/templates/service.yaml b/apps/fnux-playground/openldap/templates/service.yaml
new file mode 100644
index 0000000..8028395
--- /dev/null
+++ b/apps/fnux-playground/openldap/templates/service.yaml
@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: "{{ tpl .Values.identifier . }}"
+  labels:
+    app: openldap
+spec:
+  type: ClusterIP
+  ports:
+    - port: 389
+      name: ldap
+    - port: 636
+      name: ldaps
diff --git a/apps/fnux-playground/openldap/values.yaml b/apps/fnux-playground/openldap/values.yaml
new file mode 100644
index 0000000..9ca7b47
--- /dev/null
+++ b/apps/fnux-playground/openldap/values.yaml
@@ -0,0 +1,7 @@
+clusterDomain: "c1.k8s.ooo"
+fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"
+identifier: "{{ .Release.Name }}"
+
+ldapLogLevel: "256"
+ldapOrganisation: "ungleich glarus ag"
+ldapDomain: "{{ tpl .Values.fqdn . }}"

From a131fe805d77d53acb5abf39c5f4d5e2ef16c14f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Sat, 31 Jul 2021 16:17:15 +0200
Subject: [PATCH 07/13] Add minimal/WIP matrix-synapse setup logic

---
 apps/fnux-playground/README.md                | 44 +++++++++++++++
 .../matrix.fnux-playground.yaml               | 53 +++++++++++++++++++
 2 files changed, 97 insertions(+)
 create mode 100644 apps/fnux-playground/README.md
 create mode 100644 apps/fnux-playground/matrix.fnux-playground.yaml

diff --git a/apps/fnux-playground/README.md b/apps/fnux-playground/README.md
new file mode 100644
index 0000000..7c8d134
--- /dev/null
+++ b/apps/fnux-playground/README.md
@@ -0,0 +1,44 @@
+# Fnux's playground
+
+Tests made by Timothée for ungleich.
+
+## OpenLDAP
+
+Simple chart based on [Osixia's OpenLDAP
+image](https://github.com/osixia/docker-openldap).
+
+```
+helm install ldap1 ./openldap
+```
+
+## Matrix Synapse
+
+Matrix Homeserver setup based on [Ananace's Helm
+charts](https://github.com/osixia/docker-openldap). I exchanged a few mails
+with him, he's nice!
+
+Note: we need to wire up some network policy to firewall the various components.
+Note: there's some configuration and secret management to work on!
+Note: there's a missing bit for IPv6 support (https://gitlab.com/ananace/charts/-/merge_requests/15)
+
+```
+helm repo add ananace-charts https://ananace.gitlab.io/charts
+helm repo update
+
+helm install matrix ananace-charts/matrix-synapse --set serverName=matrix.fnux-playground.svc.c1.k8s.ooo --set wellknown.enabled=true -f matrix.fnux-playground.yaml
+```
+
+## Ingress
+
+Ingress is used by the matrix-synapse chart to distribute requests across
+synapse workers. We could do it ourselve (just generate a NGINX container from
+synapse's config) but there's already ingress logic around, which do this for
+us...
+
+```
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+helm repo update
+
+helm install ingress-nginx ingress-nginx/ingress-nginx
+```
+
diff --git a/apps/fnux-playground/matrix.fnux-playground.yaml b/apps/fnux-playground/matrix.fnux-playground.yaml
new file mode 100644
index 0000000..e9d8d80
--- /dev/null
+++ b/apps/fnux-playground/matrix.fnux-playground.yaml
@@ -0,0 +1,53 @@
+# TODO: set redis & PGSQL password, investigate workers.
+
+# Synapse configuration.
+config:
+  logLevel: INFO
+
+# Synapse persistence.
+persistence:
+  enabled: true
+  storageClass: "rook-cephfs"
+  accessMode: ReadWriteMany
+  size: 10Gi
+
+# PGSQL persistence.
+postgresql:
+  enabled: true
+  persistence:
+    storageClass: "rook-cephfs"
+    size: 16Gi
+
+# First/initial startup is slow! The synapse pod get killed before the database
+# is fully initialied if we don't explicitely wait.
+synapse:
+  livenessProbe:
+    httpGet:
+      path: /health
+      port: http
+    initialDelaySeconds: 180
+  readinessProbe:
+    httpGet:
+      path: /health
+      port: http
+    initialDelaySeconds: 180
+
+# Serve /.well-known URIs, making federation possible without adding
+# SRV-records to DNS.
+wellknown:
+  enabled: true
+
+  # Lighttpd does not bind to v6 by default - which doesn't play well in a
+  # v6-only cluster!
+  useIpv6: true
+
+  # Data served on .well-known/matrix/server.
+  # See https://matrix.org/docs/spec/server_server/latest#get-well-known-matrix-server
+  server:
+    m.server: "matrix.fnux-playground.svc.c1.k8s.oo:443"
+
+  # Data served on .well-known/matrix/client.
+  # See https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+  client:
+    m.homeserver:
+      base_url: "https://matrix.ungleich.ch"

From 68bbbba0dfbae77b1f502ca4a7af4e8a0b3a537f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 08:20:14 +0200
Subject: [PATCH 08/13] fnux-playground/openldap: set upgrade policy, fix
 service endpoints

---
 apps/fnux-playground/openldap/templates/deployment.yaml | 4 ++++
 apps/fnux-playground/openldap/templates/service.yaml    | 4 +++-
 apps/fnux-playground/openldap/values.yaml               | 1 +
 3 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/apps/fnux-playground/openldap/templates/deployment.yaml b/apps/fnux-playground/openldap/templates/deployment.yaml
index 9668566..6541d6b 100644
--- a/apps/fnux-playground/openldap/templates/deployment.yaml
+++ b/apps/fnux-playground/openldap/templates/deployment.yaml
@@ -6,6 +6,10 @@ metadata:
     app: openldap
 spec:
   replicas: 1
+  strategy:
+    # Delete old pod before starting the new one - slapd doesn't react well
+    # with two instances hitting the same database.
+    type: "Recreate"
   selector:
     matchLabels:
       app: "{{ tpl .Values.identifier . }}-openldap"
diff --git a/apps/fnux-playground/openldap/templates/service.yaml b/apps/fnux-playground/openldap/templates/service.yaml
index 8028395..f4007a3 100644
--- a/apps/fnux-playground/openldap/templates/service.yaml
+++ b/apps/fnux-playground/openldap/templates/service.yaml
@@ -1,11 +1,13 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: "{{ tpl .Values.identifier . }}"
+  name: "{{ .Release.Name }}"
   labels:
     app: openldap
 spec:
   type: ClusterIP
+  selector:
+    app: "{{ tpl .Values.identifier . }}-openldap"
   ports:
     - port: 389
       name: ldap
diff --git a/apps/fnux-playground/openldap/values.yaml b/apps/fnux-playground/openldap/values.yaml
index 9ca7b47..eb5073f 100644
--- a/apps/fnux-playground/openldap/values.yaml
+++ b/apps/fnux-playground/openldap/values.yaml
@@ -2,6 +2,7 @@ clusterDomain: "c1.k8s.ooo"
 fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"
 identifier: "{{ .Release.Name }}"
 
+# See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
 ldapLogLevel: "256"
 ldapOrganisation: "ungleich glarus ag"
 ldapDomain: "{{ tpl .Values.fqdn . }}"

From 36f37753a7459f5757ddbc4c8b1e2133fc3c880b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 08:20:58 +0200
Subject: [PATCH 09/13] fnux-playground: add configuration for
 ldap1.fnux-playground deployment

---
 apps/fnux-playground/README.md                  | 2 +-
 apps/fnux-playground/ldap1.fnux-playground.yaml | 6 ++++++
 2 files changed, 7 insertions(+), 1 deletion(-)
 create mode 100644 apps/fnux-playground/ldap1.fnux-playground.yaml

diff --git a/apps/fnux-playground/README.md b/apps/fnux-playground/README.md
index 7c8d134..562313c 100644
--- a/apps/fnux-playground/README.md
+++ b/apps/fnux-playground/README.md
@@ -8,7 +8,7 @@ Simple chart based on [Osixia's OpenLDAP
 image](https://github.com/osixia/docker-openldap).
 
 ```
-helm install ldap1 ./openldap
+helm install ldap1 ./openldap -f ldap1.fnux-playground.yaml
 ```
 
 ## Matrix Synapse
diff --git a/apps/fnux-playground/ldap1.fnux-playground.yaml b/apps/fnux-playground/ldap1.fnux-playground.yaml
new file mode 100644
index 0000000..9026f01
--- /dev/null
+++ b/apps/fnux-playground/ldap1.fnux-playground.yaml
@@ -0,0 +1,6 @@
+clusterDomain: "c1.k8s.ooo"
+
+# See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
+ldapLogLevel: "256"
+ldapOrganisation: "ungleich glarus ag"
+ldapDomain: "ungleich.ch"

From 2723af1a8e1eb6b7b6376d97bb4a092926549d02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 09:49:28 +0200
Subject: [PATCH 10/13] fnux-playground/openldap: add initial TLS/LetsEncrypt
 support

---
 apps/fnux-playground/README.md                |  6 +-
 .../openldap/templates/deployment.yaml        | 66 +++++++++++++++++--
 .../openldap/templates/pvc.yaml               | 16 ++++-
 .../openldap/templates/service.yaml           |  3 +
 apps/fnux-playground/openldap/values.yaml     |  6 ++
 5 files changed, 90 insertions(+), 7 deletions(-)

diff --git a/apps/fnux-playground/README.md b/apps/fnux-playground/README.md
index 562313c..c8c71ef 100644
--- a/apps/fnux-playground/README.md
+++ b/apps/fnux-playground/README.md
@@ -5,9 +5,13 @@ Tests made by Timothée for ungleich.
 ## OpenLDAP
 
 Simple chart based on [Osixia's OpenLDAP
-image](https://github.com/osixia/docker-openldap).
+image](https://github.com/osixia/docker-openldap). A TLS certificate is
+automatically generated via Let'sEncrypt, but renewal is not handled yet.
+
+TODO: handle TLS certificate renewal.
 
 ```
+kubectl create secret generic ldap1-openldap --from-literal=LDAP_ADMIN_PASSWORD=secretsecretsectet
 helm install ldap1 ./openldap -f ldap1.fnux-playground.yaml
 ```
 
diff --git a/apps/fnux-playground/openldap/templates/deployment.yaml b/apps/fnux-playground/openldap/templates/deployment.yaml
index 6541d6b..7bd9e91 100644
--- a/apps/fnux-playground/openldap/templates/deployment.yaml
+++ b/apps/fnux-playground/openldap/templates/deployment.yaml
@@ -18,10 +18,20 @@ spec:
       labels:
         app: "{{ tpl .Values.identifier . }}-openldap"
     spec:
+      initContainers:
+        - name: wait-for-cert
+          image: busybox
+          command:
+            - "sh"
+            - "-c"
+            - "until ls /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; do sleep 5; done"
+          volumeMounts:
+            - name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
+              mountPath: /etc/letsencrypt
       containers:
         - name: "openldap"
           image: "osixia/openldap:{{ .Chart.AppVersion }}"
-          args: ["--loglevel", "info"]
+          args: ["--loglevel", "debug"]
           ports:
             - name: ldap
               containerPort: 389
@@ -51,8 +61,16 @@ spec:
             - name: LDAP_ADMIN_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: "{{ tpl .Values.identifier . }}-openldap"
-                  key: LDAP_ADMIN_PASSWORD
+                  name: "{{ tpl .Values.ldapAdminPasswordSecretRef . }}"
+                  key: "{{ tpl .Values.ldapAdminPasswordSecretKey . }}"
+            - name: LDAP_TLS_CRT_FILENAME
+              value: "live/{{ tpl .Values.fqdn . }}/cert.pem"
+            - name: LDAP_TLS_KEY_FILENAME
+              value: "live/{{ tpl .Values.fqdn . }}/privkey.pem"
+            - name: LDAP_TLS_CA_CRT_FILENAME
+              value: "live/{{ tpl .Values.fqdn . }}/fullchain.pem"
+            - name: LDAP_TLS_VERIFY_CLIENT
+              value: "try"
           volumeMounts:
             - name: "{{ tpl .Values.identifier . }}-openldap-data"
               mountPath: "/etc/ldap/slapd.d"
@@ -60,10 +78,48 @@ spec:
             - name: "{{ tpl .Values.identifier . }}-openldap-data"
               mountPath: "/var/lib/ldap"
               subPath: database
-            - name: "{{ tpl .Values.identifier . }}-openldap-data"
+            - name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
               mountPath: /container/service/slapd/assets/certs
-              subPath: certs
       volumes:
         - name: "{{ tpl .Values.identifier . }}-openldap-data"
           persistentVolumeClaim:
             claimName: "{{ tpl .Values.identifier . }}-openldap-data"
+        - name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
+          persistentVolumeClaim:
+            claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ tpl .Values.identifier . }}-getcert
+spec:
+  template:
+    metadata:
+      labels:
+        app: {{ tpl .Values.identifier . }}-openldap
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: certbot
+        image: ungleich/ungleich-certbot
+        ports:
+          - containerPort: 80
+        env:
+          - name: ONLYGETCERT
+            value: "yes"
+          - name: DOMAIN
+            value: "{{ tpl .Values.fqdn . }}"
+          - name: EMAIL
+            value: "{{ .Values.letsencryptEmail }}"
+          {{ if eq .Values.letsencryptStaging "no" }}
+          - name: STAGING
+            value: "no"
+          {{ end }}
+        volumeMounts:
+          - name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
+            mountPath: /etc/letsencrypt
+      volumes:
+        - name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
+          persistentVolumeClaim:
+            claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs
+  backoffLimit: 3
diff --git a/apps/fnux-playground/openldap/templates/pvc.yaml b/apps/fnux-playground/openldap/templates/pvc.yaml
index 66d55ad..360eb6f 100644
--- a/apps/fnux-playground/openldap/templates/pvc.yaml
+++ b/apps/fnux-playground/openldap/templates/pvc.yaml
@@ -4,8 +4,22 @@ metadata:
   name: "{{ tpl .Values.identifier . }}-openldap-data"
 spec:
   accessModes:
-    - ReadWriteMany
+    - ReadWriteOnce
   resources:
     requests:
       storage: 1Gi
   storageClassName: rook-cephfs
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: "{{ tpl .Values.identifier . }}-letsencrypt-certs"
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Mi
+  storageClassName: rook-cephfs
+
+
diff --git a/apps/fnux-playground/openldap/templates/service.yaml b/apps/fnux-playground/openldap/templates/service.yaml
index f4007a3..43e86d2 100644
--- a/apps/fnux-playground/openldap/templates/service.yaml
+++ b/apps/fnux-playground/openldap/templates/service.yaml
@@ -13,3 +13,6 @@ spec:
       name: ldap
     - port: 636
       name: ldaps
+    # Required for TLS certificate generation via LetsEncrypt.
+    - port: 80
+      name: http
diff --git a/apps/fnux-playground/openldap/values.yaml b/apps/fnux-playground/openldap/values.yaml
index eb5073f..f0c48f9 100644
--- a/apps/fnux-playground/openldap/values.yaml
+++ b/apps/fnux-playground/openldap/values.yaml
@@ -6,3 +6,9 @@ identifier: "{{ .Release.Name }}"
 ldapLogLevel: "256"
 ldapOrganisation: "ungleich glarus ag"
 ldapDomain: "{{ tpl .Values.fqdn . }}"
+ldapAdminPasswordSecretRef: "{{ tpl .Values.identifier . }}-openldap"
+ldapAdminPasswordSecretKey: "LDAP_ADMIN_PASSWORD"
+
+# TLS certificate generation.
+letsencryptEmail: "technik@ungleich.ch"
+letsencryptStaging: "no"

From 54fa93a422e1803c0c077fec86e5c0bd8ee0c5ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 18:08:56 +0200
Subject: [PATCH 11/13] fnux-playground: add TLS and replication to openldap
 chart

---
 apps/fnux-playground/README.md                |  3 ++
 .../ldap1.fnux-playground.yaml                |  6 ----
 .../ldapN.fnux-playground.yaml                | 10 +++++++
 apps/fnux-playground/openldap/Chart.yaml      |  2 +-
 .../openldap/templates/deployment.yaml        | 29 ++++++++++++++-----
 apps/fnux-playground/openldap/values.yaml     | 19 +++++++-----
 6 files changed, 47 insertions(+), 22 deletions(-)
 delete mode 100644 apps/fnux-playground/ldap1.fnux-playground.yaml
 create mode 100644 apps/fnux-playground/ldapN.fnux-playground.yaml

diff --git a/apps/fnux-playground/README.md b/apps/fnux-playground/README.md
index c8c71ef..c406b84 100644
--- a/apps/fnux-playground/README.md
+++ b/apps/fnux-playground/README.md
@@ -9,6 +9,9 @@ image](https://github.com/osixia/docker-openldap). A TLS certificate is
 automatically generated via Let'sEncrypt, but renewal is not handled yet.
 
 TODO: handle TLS certificate renewal.
+NOTE: replication with the osixia image is somewhat broken, see:
+	https://github.com/osixia/docker-openldap/issues/203
+	-> Worked around the issue with https://github.com/ungleich/docker-openldap/commit/3c7c9ece1e67bce0bfe1fdb66a63f5c8c59359f4
 
 ```
 kubectl create secret generic ldap1-openldap --from-literal=LDAP_ADMIN_PASSWORD=secretsecretsectet
diff --git a/apps/fnux-playground/ldap1.fnux-playground.yaml b/apps/fnux-playground/ldap1.fnux-playground.yaml
deleted file mode 100644
index 9026f01..0000000
--- a/apps/fnux-playground/ldap1.fnux-playground.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
-clusterDomain: "c1.k8s.ooo"
-
-# See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
-ldapLogLevel: "256"
-ldapOrganisation: "ungleich glarus ag"
-ldapDomain: "ungleich.ch"
diff --git a/apps/fnux-playground/ldapN.fnux-playground.yaml b/apps/fnux-playground/ldapN.fnux-playground.yaml
new file mode 100644
index 0000000..da29629
--- /dev/null
+++ b/apps/fnux-playground/ldapN.fnux-playground.yaml
@@ -0,0 +1,10 @@
+clusterDomain: "c1.k8s.ooo"
+
+ldap:
+  # See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
+  logLevel: "256"
+  oganisation: "ungleich glarus ag"
+  domain: "ungleich.ch"
+  adminPasswordSecretRef: "ldap-openldap"
+  enableReplication: "true"
+  replicationHosts: "#PYTHON2BASH:['ldaps://ldap1.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}','ldaps://ldap2.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}']"
diff --git a/apps/fnux-playground/openldap/Chart.yaml b/apps/fnux-playground/openldap/Chart.yaml
index 1b45324..137d4fc 100644
--- a/apps/fnux-playground/openldap/Chart.yaml
+++ b/apps/fnux-playground/openldap/Chart.yaml
@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "1.5.0"
+appVersion: "1.5.0-serverid-hostname-fallback-2"
diff --git a/apps/fnux-playground/openldap/templates/deployment.yaml b/apps/fnux-playground/openldap/templates/deployment.yaml
index 7bd9e91..ca9be71 100644
--- a/apps/fnux-playground/openldap/templates/deployment.yaml
+++ b/apps/fnux-playground/openldap/templates/deployment.yaml
@@ -30,8 +30,8 @@ spec:
               mountPath: /etc/letsencrypt
       containers:
         - name: "openldap"
-          image: "osixia/openldap:{{ .Chart.AppVersion }}"
-          args: ["--loglevel", "debug"]
+          image: "ungleich/openldap:{{ .Chart.AppVersion }}"
+          args: ["--loglevel", "trace"]
           ports:
             - name: ldap
               containerPort: 389
@@ -53,16 +53,21 @@ spec:
             - name: HOSTNAME
               value: "{{ tpl .Values.fqdn . }}"
             - name: LDAP_LOG_LEVEL
-              value: "{{ tpl .Values.ldapLogLevel . }}"
+              value: "{{ tpl .Values.ldap.logLevel . }}"
             - name: LDAP_ORGANISATION
-              value: "{{ tpl .Values.ldapOrganisation . }}"
+              value: "{{ tpl .Values.ldap.organisation . }}"
             - name: LDAP_DOMAIN
-              value: "{{ tpl .Values.ldapDomain . }}"
+              value: "{{ tpl .Values.ldap.domain . }}"
             - name: LDAP_ADMIN_PASSWORD
               valueFrom:
                 secretKeyRef:
-                  name: "{{ tpl .Values.ldapAdminPasswordSecretRef . }}"
-                  key: "{{ tpl .Values.ldapAdminPasswordSecretKey . }}"
+                  name: "{{ tpl .Values.ldap.adminPasswordSecretRef . }}"
+                  key: "{{ tpl .Values.ldap.adminPasswordSecretKey . }}"
+            - name: LDAP_CONFIG_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: "{{ tpl .Values.ldap.adminPasswordSecretRef . }}"
+                  key: "{{ tpl .Values.ldap.adminPasswordSecretKey . }}"
             - name: LDAP_TLS_CRT_FILENAME
               value: "live/{{ tpl .Values.fqdn . }}/cert.pem"
             - name: LDAP_TLS_KEY_FILENAME
@@ -71,6 +76,14 @@ spec:
               value: "live/{{ tpl .Values.fqdn . }}/fullchain.pem"
             - name: LDAP_TLS_VERIFY_CLIENT
               value: "try"
+            - name: LDAP_REPLICATION
+              value: "{{ .Values.ldap.enableReplication }}"
+            - name: LDAP_REPLICATION_HOSTS
+              value: "{{ tpl .Values.ldap.replicationHosts . }}"
+            - name: LDAP_REPLICATION_CONFIG_SYNCPROV
+              value: "{{ tpl .Values.ldap.replicationConfigSyncprov . }}"
+            - name: LDAP_REPLICATION_DB_SYNCPROV
+              value: "{{ tpl .Values.ldap.replicationDbSyncprov . }}"
           volumeMounts:
             - name: "{{ tpl .Values.identifier . }}-openldap-data"
               mountPath: "/etc/ldap/slapd.d"
@@ -111,7 +124,7 @@ spec:
             value: "{{ tpl .Values.fqdn . }}"
           - name: EMAIL
             value: "{{ .Values.letsencryptEmail }}"
-          {{ if eq .Values.letsencryptStaging "no" }}
+          {{ if not .Values.letsencryptStaging }}
           - name: STAGING
             value: "no"
           {{ end }}
diff --git a/apps/fnux-playground/openldap/values.yaml b/apps/fnux-playground/openldap/values.yaml
index f0c48f9..52e9196 100644
--- a/apps/fnux-playground/openldap/values.yaml
+++ b/apps/fnux-playground/openldap/values.yaml
@@ -2,13 +2,18 @@ clusterDomain: "c1.k8s.ooo"
 fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"
 identifier: "{{ .Release.Name }}"
 
-# See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
-ldapLogLevel: "256"
-ldapOrganisation: "ungleich glarus ag"
-ldapDomain: "{{ tpl .Values.fqdn . }}"
-ldapAdminPasswordSecretRef: "{{ tpl .Values.identifier . }}-openldap"
-ldapAdminPasswordSecretKey: "LDAP_ADMIN_PASSWORD"
+ldap:
+  # See https://www.openldap.org/doc/admin24/slapdconf2.html section 5.2.1.2;
+  logLevel: "256"
+  organisation: "ungleich glarus ag"
+  domain: "{{ tpl .Values.fqdn . }}"
+  adminPasswordSecretRef: "{{ tpl .Values.identifier . }}-openldap"
+  adminPasswordSecretKey: "LDAP_ADMIN_PASSWORD"
+  enableReplication: false
+  replicationHosts: ""
+  replicationConfigSyncprov: 'binddn=\"cn=admin,cn=config\" bindmethod=simple credentials=$$LDAP_CONFIG_PASSWORD searchbase=\"cn=config\" type=refreshAndPersist retry=\"60 +\" timeout=1 starttls=no'
+  replicationDbSyncprov: 'binddn=\"cn=admin,$$LDAP_BASE_DN\" bindmethod=simple credentials=$$LDAP_ADMIN_PASSWORD searchbase=\"$$LDAP_BASE_DN\" type=refreshAndPersist interval=00:00:00:10 retry=\"60 +\" timeout=1 starttls=no'
 
 # TLS certificate generation.
 letsencryptEmail: "technik@ungleich.ch"
-letsencryptStaging: "no"
+letsencryptStaging: false

From 3bf9d54ff07954bdc65ec5dbff3efff96c1e99c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 18:12:25 +0200
Subject: [PATCH 12/13] fnux-playground: expand matrix configuration

---
 .../matrix.fnux-playground.yaml               | 89 ++++++++++++++-----
 1 file changed, 69 insertions(+), 20 deletions(-)

diff --git a/apps/fnux-playground/matrix.fnux-playground.yaml b/apps/fnux-playground/matrix.fnux-playground.yaml
index e9d8d80..5063b44 100644
--- a/apps/fnux-playground/matrix.fnux-playground.yaml
+++ b/apps/fnux-playground/matrix.fnux-playground.yaml
@@ -1,26 +1,36 @@
-# TODO: set redis & PGSQL password, investigate workers.
+# Note: as of writing we can't template the variables of this file, although
+# I'm pretty sure upstream would accept a patch for this.
 
-# Synapse configuration.
+# Shared variables.
+clusterName: "c2.k8s.ooo"
+
+# The Matrix domain name, this is what will be used for the domain part in
+# your MXIDs.
+serverName: "matrix.fnux-playground.svc.c2.k8s.ooo"
+
+# The public Matrix server name, this will be used for any public URLs
+# in config as well as for client API links in the ingress.
+publicServerName: "matrix.fnux-playground.svc.c2.k8s.ooo"
+
+# Generic configuration that apply to mixed components.
 config:
+  # Log level for Synapse and all modules.
   logLevel: INFO
 
-# Synapse persistence.
-persistence:
-  enabled: true
-  storageClass: "rook-cephfs"
-  accessMode: ReadWriteMany
-  size: 10Gi
-
-# PGSQL persistence.
-postgresql:
-  enabled: true
-  persistence:
-    storageClass: "rook-cephfs"
-    size: 16Gi
-
-# First/initial startup is slow! The synapse pod get killed before the database
-# is fully initialied if we don't explicitely wait.
+# Configuration to apply to the main Synapse pod.
 synapse:
+  ## Only really applicable when the deployment has an RWO PV attached (e.g. when media repository
+  ## is enabled for the main Synapse pod)
+  ## Since replicas = 1, an update can get "stuck", as the previous pod remains attached to the
+  ## PV, and the "incoming" pod can never start. Changing the strategy to "Recreate" will
+  ## terminate the single previous pod, so that the new, incoming pod can attach to the PV
+  ##
+  strategy:
+    type: RollingUpdate
+  # First/initial startup is slow! The synapse pod get killed before the
+  # database is fully initialied if we don't explicitely wait.
+  # XXX: we should probably use a startupProbe, but this need to be patched
+  # upstream.
   livenessProbe:
     httpGet:
       path: /health
@@ -32,6 +42,22 @@ synapse:
       port: http
     initialDelaySeconds: 180
 
+# Configuration for handling Synapse workers, which are useful for handling
+# high-load deployments.
+#
+# More information is available at;
+# https://github.com/matrix-org/synapse/blob/master/docs/workers.md
+#
+# workers: ...
+
+# Persistence configuration for the media repository function.  This PVC will
+# be mounted in either Synapse or a media_repo worker.
+persistence:
+  enabled: true
+  storageClass: "rook-cephfs"
+  accessMode: ReadWriteMany
+  size: 10Gi
+
 # Serve /.well-known URIs, making federation possible without adding
 # SRV-records to DNS.
 wellknown:
@@ -44,10 +70,33 @@ wellknown:
   # Data served on .well-known/matrix/server.
   # See https://matrix.org/docs/spec/server_server/latest#get-well-known-matrix-server
   server:
-    m.server: "matrix.fnux-playground.svc.c1.k8s.oo:443"
+    m.server: "matrix.fnux-playground.svc.c2.k8s.ooo"
 
   # Data served on .well-known/matrix/client.
   # See https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
   client:
     m.homeserver:
-      base_url: "https://matrix.ungleich.ch"
+      base_url: "https://matrix.fnux-playground.svc.c2.k8s.ooo"
+
+# PGSQL database server configuration.
+postgresql:
+  enabled: true
+  postgresqlPassword: "secret"
+  postgresqlUsername: synapse
+  postgresqlDatabase: synapse
+  persistence:
+    storageClass: "rook-cephfs"
+    size: 16Gi
+
+## Redis server for use with workers/sharding.
+redis:
+  enabled: true
+  usePassword: true
+  password: "secret"
+
+# The K8s ingress configuration, this will be quite heavily used in order to
+# set up all routing necessary for use with a sharded Synapse instance. If
+# you're not using a Ingress compatible K8s ingress, you will need to set up
+# your own routing instead.
+ingress:
+  enabled: true

From 4f9678be3a3dc69c811b1017142aa1bcf8dd2e93 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= <timothee.floure@posteo.net>
Date: Mon, 2 Aug 2021 18:16:26 +0200
Subject: [PATCH 13/13] fnux-playground: refresh openldap instructions/example

---
 apps/fnux-playground/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/fnux-playground/README.md b/apps/fnux-playground/README.md
index c406b84..5de1ac4 100644
--- a/apps/fnux-playground/README.md
+++ b/apps/fnux-playground/README.md
@@ -15,7 +15,8 @@ NOTE: replication with the osixia image is somewhat broken, see:
 
 ```
 kubectl create secret generic ldap1-openldap --from-literal=LDAP_ADMIN_PASSWORD=secretsecretsectet
-helm install ldap1 ./openldap -f ldap1.fnux-playground.yaml
+helm install ldap1 ./openldap -f ldapN.fnux-playground.yaml
+helm install ldap2 ./openldap -f ldapN.fnux-playground.yaml
 ```
 
 ## Matrix Synapse