Merge branch 'master' of code.ungleich.ch:ungleich-public/ungleich-k8s

README.md

@@ -13,13 +13,18 @@ This project is testing, deploying and using IPv6 only k8s clusters.
 * ceph with rook (cephfs, rbd)
 * letsencrypt (nginx, certbot, homemade)
 * k8s test on arm64
+* CI/CD using flux
+* Chart repository (chartmuseum)
+* Git repository (gitea)
 
 ## Not (yet) working or tested
 
+* proxy for pulling images only
+  * configure a proxy on crio
+  * setup a proxy in the cluster (?)
 * virtualisation (VMs, kubevirt)
 * network policies
-* prometheus in the cluster
-* -argocd (?) for CI and upgrades- using flux
+* Prometheus for the cluster
 * Maybe LoadBalancer support (our ClusterIP already does that though)
 * (Other) DNS entrys for services
 * Internal backup / snapshots
@@ -242,6 +247,8 @@ referes to an external load balancer that
 
 ## Secrets
 
+### Generating them inside the cluster
+
 Handled via https://github.com/mittwald/kubernetes-secret-generator
 
 ```
@@ -263,7 +270,36 @@ data:
   username: c29tZXVzZXI=
 ```
 
-This will add a password into it. Password only will
+* Advantage: passwords are only in the cluster
+* Disadvantage: passwords are only in the cluster
+
+## CI/CD
+
+### What we want
+
+* Package everything into one git repository (charts, kustomize, etc.)
+* Be usable for multiple clusters
+* Easily apply cross cluster
+
+### What we don't want / what is problematic
+
+* Uploading charts to something like chartmuseum
+  * Is redundant - we have a version in git
+  * Is manual (could probably be automated)
+
+### ArgoCD
+
+Looks too big, too complex, too complicated.
+
+### FluxCD2
+
+Looks ok, handling of helm is ok, but does not feel intuitive. Seems
+to be more orientated on "kustomizing helm charts".
+
+### Helmfile
+
+[helmfile](https://github.com/roboll/helmfile/) seems to do most of
+what we need.
 
 ## The IPv4 "problem"
 

apps/chartmuseum/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

apps/chartmuseum/Chart.yaml

@@ -0,0 +1,11 @@
+apiVersion: v2
+name: ungleich-chartmuseum
+description: Chartmuseum for the ungleich infrastructure
+
+version: 0.1.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "0.13.1"

apps/chartmuseum/README.md

@@ -0,0 +1,15 @@
+## Missing
+
+* SSL/TLS
+  * Letsencrypt on port 80
+  * Chartmuseum on 443
+  * Reload?
+* Authentication
+  * Secret -> generate
+  * Secret -> sops
+
+
+## Done
+
+* Basic chartmuseum
+* helm push works

apps/chartmuseum/templates/deployment.yaml

@@ -0,0 +1,172 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .Release.Name }}-chartmuseum
+spec:
+  selector:
+    matchLabels:
+      app: {{ .Release.Name }}-chartmuseum
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: {{ .Release.Name }}-chartmuseum
+        use-as-service: {{ .Release.Name }}
+    spec:
+      # Wait before trying to start any container
+      initcontainers:
+        - name: wait-for-cert
+          image: busybox
+          command:
+            - sh
+            - -c
+            - until ls /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; do sleep 5; done
+          volumeMounts:
+            - name: etcletsencrypt
+              mountPath: "/etc/letsencrypt"
+      containers:
+        - name: certbot
+          image: ungleich/ungleich-certbot
+          imagePullPolicy: Always
+          ports:
+            - containerPort: 80
+          env:
+          - name: ONLYRENEWCERTS
+            value: "yes"
+          - name: DOMAIN
+            value: "{{ tpl .Values.fqdn . }}"
+          - name: EMAIL
+            value: "{{ .Values.email }}"
+          {{ if eq .Values.letsencryptStaging "no" }}
+          - name: STAGING
+            value: "no"
+          {{ end }}
+          volumeMounts:
+            - name: etcletsencrypt
+              mountPath: "/etc/letsencrypt"
+        - name: chartmuseum
+          image: ghcr.io/helm/chartmuseum:v{{ .Chart.AppVersion }}
+          ports:
+            - containerPort: 443
+          args:
+            - --tls-cert=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem
+            - --tls-key=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem
+          env:
+            - name: STORAGE
+              value: "local"
+            - name: STORAGE_LOCAL_ROOTDIR
+              value: "/charts"
+          volumeMounts:
+            - name: etcletsencrypt
+              mountPath: "/etc/letsencrypt"
+            - name: data
+              mountPath: "/charts"
+      volumes:
+        - name: etcletsencrypt
+          persistentVolumeClaim:
+            claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ tpl .Values.identifier . }}-data
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ tpl .Values.identifier . }}
+  labels:
+    app: {{ tpl .Values.identifier . }}
+spec:
+  type: ClusterIP
+  ports:
+   - port: 80
+     name: http
+   - port: 443
+     name: https
+  selector:
+   use-as-service: {{ .Release.Name }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ tpl .Values.identifier . }}-letsencrypt-certs
+spec:
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 50Mi
+  storageClassName: rook-cephfs
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ tpl .Values.identifier . }}-data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+  storageClassName: rook-ceph-block
+# ---
+# apiVersion: v1
+# kind: ConfigMap
+# metadata:
+#   name: {{ tpl .Values.identifier . }}-nginx-config
+# data:
+#   default.conf: |
+#     server {
+#         listen 443 ssl;
+#         listen [::]:443 ssl;
+
+#         server_name {{ tpl .Values.fqdn . }};
+
+#         ssl_certificate      /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem;
+#         ssl_certificate_key  /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem;
+
+#         client_max_body_size 256m;
+
+#         location / {
+#             proxy_pass http://localhost:3000;
+#         }
+#     }
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ tpl .Values.identifier . }}-getcert
+spec:
+  template:
+    metadata:
+      labels:
+        app: certbot-letsencrypt-getcert
+        use-as-service: {{ .Release.Name }}
+    spec:
+      restartPolicy: Never
+      containers:
+      - name: certbot
+        image: ungleich/ungleich-certbot
+        imagePullPolicy: Always
+
+        ports:
+          - containerPort: 80
+        env:
+          - name: ONLYGETCERT
+            value: "yes"
+          - name: DOMAIN
+            value: "{{ tpl .Values.fqdn . }}"
+          - name: EMAIL
+            value: "{{ .Values.email }}"
+          {{ if eq .Values.letsencryptStaging "no" }}
+          - name: STAGING
+            value: "no"
+          {{ end }}
+        volumeMounts:
+          - name: etcletsencrypt
+            mountPath: "/etc/letsencrypt"
+      volumes:
+        - name: etcletsencrypt
+          persistentVolumeClaim:
+            claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs
+  backoffLimit: 3

apps/chartmuseum/templates/tests/test-connection.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: "{{ tpl .Values.identifier . }}-test-connection"
+  annotations:
+    "helm.sh/hook": test
+spec:
+  containers:
+    - name: wget
+      image: busybox
+      command: ['wget']
+      args: ['https://{{ tpl .Values.fqdn . }}']
+  restartPolicy: Never

apps/chartmuseum/values.yaml

@@ -0,0 +1,6 @@
+clusterDomain: c2.k8s.ooo
+email: technik@ungleich.ch
+letsencryptStaging: "yes"
+
+identifier: "{{ .Release.Name }}"
+fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}"

apps/knotdns/Chart.yaml

@@ -1,5 +1,6 @@
 apiVersion: v2
 name: knotdns
-description: DNS Auth
+description: Authoritative DNS server
 
-version: 0.1.0
+version: 0.1.1
+appVersion: "3.0"

apps/knotdns/templates/deployment.yaml

@@ -19,7 +19,7 @@ spec:
     spec:
       containers:
         - name: knot
-          image: cznic/knot:3.0
+          image: "cznic/knot:{{ .Chart.AppVersion }}"
           ports:
             - name: udp-53
               containerPort: 53