diff --git a/README.md b/README.md index 649637c..f14c907 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,18 @@ This project is testing, deploying and using IPv6 only k8s clusters. * ceph with rook (cephfs, rbd) * letsencrypt (nginx, certbot, homemade) * k8s test on arm64 +* CI/CD using flux +* Chart repository (chartmuseum) +* Git repository (gitea) ## Not (yet) working or tested +* proxy for pulling images only + * configure a proxy on crio + * setup a proxy in the cluster (?) * virtualisation (VMs, kubevirt) * network policies -* prometheus in the cluster -* -argocd (?) for CI and upgrades- using flux +* Prometheus for the cluster * Maybe LoadBalancer support (our ClusterIP already does that though) * (Other) DNS entrys for services * Internal backup / snapshots @@ -242,6 +247,8 @@ referes to an external load balancer that ## Secrets +### Generating them inside the cluster + Handled via https://github.com/mittwald/kubernetes-secret-generator ``` @@ -263,7 +270,36 @@ data: username: c29tZXVzZXI= ``` -This will add a password into it. Password only will +* Advantage: passwords are only in the cluster +* Disadvantage: passwords are only in the cluster + +## CI/CD + +### What we want + +* Package everything into one git repository (charts, kustomize, etc.) +* Be usable for multiple clusters +* Easily apply cross cluster + +### What we don't want / what is problematic + +* Uploading charts to something like chartmuseum + * Is redundant - we have a version in git + * Is manual (could probably be automated) + +### ArgoCD + +Looks too big, too complex, too complicated. + +### FluxCD2 + +Looks ok, handling of helm is ok, but does not feel intuitive. Seems +to be more orientated on "kustomizing helm charts". + +### Helmfile + +[helmfile](https://github.com/roboll/helmfile/) seems to do most of +what we need. ## The IPv4 "problem" diff --git a/apps/chartmuseum/.helmignore b/apps/chartmuseum/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/apps/chartmuseum/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/apps/chartmuseum/Chart.yaml b/apps/chartmuseum/Chart.yaml new file mode 100644 index 0000000..84e2f11 --- /dev/null +++ b/apps/chartmuseum/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: ungleich-chartmuseum +description: Chartmuseum for the ungleich infrastructure + +version: 0.1.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.13.1" diff --git a/apps/chartmuseum/README.md b/apps/chartmuseum/README.md new file mode 100644 index 0000000..080001d --- /dev/null +++ b/apps/chartmuseum/README.md @@ -0,0 +1,15 @@ +## Missing + +* SSL/TLS + * Letsencrypt on port 80 + * Chartmuseum on 443 + * Reload? +* Authentication + * Secret -> generate + * Secret -> sops + + +## Done + +* Basic chartmuseum +* helm push works diff --git a/apps/chartmuseum/templates/deployment.yaml b/apps/chartmuseum/templates/deployment.yaml new file mode 100644 index 0000000..f811498 --- /dev/null +++ b/apps/chartmuseum/templates/deployment.yaml @@ -0,0 +1,172 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-chartmuseum +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-chartmuseum + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }}-chartmuseum + use-as-service: {{ .Release.Name }} + spec: + # Wait before trying to start any container + initcontainers: + - name: wait-for-cert + image: busybox + command: + - sh + - -c + - until ls /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; do sleep 5; done + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + containers: + - name: certbot + image: ungleich/ungleich-certbot + imagePullPolicy: Always + ports: + - containerPort: 80 + env: + - name: ONLYRENEWCERTS + value: "yes" + - name: DOMAIN + value: "{{ tpl .Values.fqdn . }}" + - name: EMAIL + value: "{{ .Values.email }}" + {{ if eq .Values.letsencryptStaging "no" }} + - name: STAGING + value: "no" + {{ end }} + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + - name: chartmuseum + image: ghcr.io/helm/chartmuseum:v{{ .Chart.AppVersion }} + ports: + - containerPort: 443 + args: + - --tls-cert=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem + - --tls-key=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem + env: + - name: STORAGE + value: "local" + - name: STORAGE_LOCAL_ROOTDIR + value: "/charts" + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + - name: data + mountPath: "/charts" + volumes: + - name: etcletsencrypt + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs + - name: data + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tpl .Values.identifier . }} + labels: + app: {{ tpl .Values.identifier . }} +spec: + type: ClusterIP + ports: + - port: 80 + name: http + - port: 443 + name: https + selector: + use-as-service: {{ .Release.Name }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ tpl .Values.identifier . }}-letsencrypt-certs +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Mi + storageClassName: rook-cephfs +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ tpl .Values.identifier . }}-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: rook-ceph-block +# --- +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: {{ tpl .Values.identifier . }}-nginx-config +# data: +# default.conf: | +# server { +# listen 443 ssl; +# listen [::]:443 ssl; + +# server_name {{ tpl .Values.fqdn . }}; + +# ssl_certificate /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; +# ssl_certificate_key /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem; + +# client_max_body_size 256m; + +# location / { +# proxy_pass http://localhost:3000; +# } +# } +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ tpl .Values.identifier . }}-getcert +spec: + template: + metadata: + labels: + app: certbot-letsencrypt-getcert + use-as-service: {{ .Release.Name }} + spec: + restartPolicy: Never + containers: + - name: certbot + image: ungleich/ungleich-certbot + imagePullPolicy: Always + + ports: + - containerPort: 80 + env: + - name: ONLYGETCERT + value: "yes" + - name: DOMAIN + value: "{{ tpl .Values.fqdn . }}" + - name: EMAIL + value: "{{ .Values.email }}" + {{ if eq .Values.letsencryptStaging "no" }} + - name: STAGING + value: "no" + {{ end }} + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + volumes: + - name: etcletsencrypt + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs + backoffLimit: 3 diff --git a/apps/chartmuseum/templates/tests/test-connection.yaml b/apps/chartmuseum/templates/tests/test-connection.yaml new file mode 100644 index 0000000..85cbdda --- /dev/null +++ b/apps/chartmuseum/templates/tests/test-connection.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ tpl .Values.identifier . }}-test-connection" + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['https://{{ tpl .Values.fqdn . }}'] + restartPolicy: Never diff --git a/apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz b/apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz new file mode 100644 index 0000000..d592a80 Binary files /dev/null and b/apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz differ diff --git a/apps/chartmuseum/values.yaml b/apps/chartmuseum/values.yaml new file mode 100644 index 0000000..bdf3229 --- /dev/null +++ b/apps/chartmuseum/values.yaml @@ -0,0 +1,6 @@ +clusterDomain: c2.k8s.ooo +email: technik@ungleich.ch +letsencryptStaging: "yes" + +identifier: "{{ .Release.Name }}" +fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" diff --git a/apps/knotdns/Chart.yaml b/apps/knotdns/Chart.yaml index d2d962b..907aefa 100644 --- a/apps/knotdns/Chart.yaml +++ b/apps/knotdns/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v2 name: knotdns -description: DNS Auth +description: Authoritative DNS server -version: 0.1.0 +version: 0.1.1 +appVersion: "3.0" diff --git a/apps/knotdns/templates/deployment.yaml b/apps/knotdns/templates/deployment.yaml index 2eeaf29..2c20348 100644 --- a/apps/knotdns/templates/deployment.yaml +++ b/apps/knotdns/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: knot - image: cznic/knot:3.0 + image: "cznic/knot:{{ .Chart.AppVersion }}" ports: - name: udp-53 containerPort: 53