From da2e9a0fa9de2a4fe69ae3b2f459699616576ef9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Jul 2021 23:05:30 +0200 Subject: [PATCH 1/6] [app] add chartmuseum --- apps/chartmuseum/.helmignore | 23 ++++ apps/chartmuseum/Chart.yaml | 11 ++ apps/chartmuseum/README.md | 6 + apps/chartmuseum/templates/deployment.yaml | 103 ++++++++++++++++++ .../templates/tests/test-connection.yaml | 13 +++ .../ungleich-chartmuseum-0.1.0.tgz | Bin 0 -> 1633 bytes apps/chartmuseum/values.yaml | 6 + 7 files changed, 162 insertions(+) create mode 100644 apps/chartmuseum/.helmignore create mode 100644 apps/chartmuseum/Chart.yaml create mode 100644 apps/chartmuseum/README.md create mode 100644 apps/chartmuseum/templates/deployment.yaml create mode 100644 apps/chartmuseum/templates/tests/test-connection.yaml create mode 100644 apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz create mode 100644 apps/chartmuseum/values.yaml diff --git a/apps/chartmuseum/.helmignore b/apps/chartmuseum/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/apps/chartmuseum/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/apps/chartmuseum/Chart.yaml b/apps/chartmuseum/Chart.yaml new file mode 100644 index 0000000..4eef696 --- /dev/null +++ b/apps/chartmuseum/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +name: ungleich-chartmuseum +description: Chartmuseum for the ungleich infrastructure + +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.13.1" diff --git a/apps/chartmuseum/README.md b/apps/chartmuseum/README.md new file mode 100644 index 0000000..7d68f32 --- /dev/null +++ b/apps/chartmuseum/README.md @@ -0,0 +1,6 @@ +## Missing + +- db secret generation (sops?) +- SMTP settings / secrets (ungleich mail + sops?) +- Exposing sizes in value.yaml (db, gitea) + - Maybe reducing to 1 PVC? diff --git a/apps/chartmuseum/templates/deployment.yaml b/apps/chartmuseum/templates/deployment.yaml new file mode 100644 index 0000000..592deee --- /dev/null +++ b/apps/chartmuseum/templates/deployment.yaml @@ -0,0 +1,103 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-chartmuseum +spec: + selector: + matchLabels: + app: {{ .Release.Name }}-chartmuseum + replicas: 1 + template: + metadata: + labels: + app: {{ .Release.Name }}-chartmuseum + use-as-service: {{ .Release.Name }} + spec: + containers: + - name: chartmuseum + image: ghcr.io/helm/chartmuseum:v0.13.1 + ports: + - containerPort: 8080 + # args: + # - --tls-cert=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem + # - --tls-key=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem + env: + - name: STORAGE + value: "local" + - name: STORAGE_LOCAL_ROOTDIR + value: "/charts" + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + - name: data + mountPath: "/charts" + volumes: + - name: etcletsencrypt + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs + - name: data + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-data +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tpl .Values.identifier . }} + labels: + app: {{ tpl .Values.identifier . }} +spec: + type: ClusterIP + ports: + - port: 8080 + name: http + - port: 443 + name: https + selector: + use-as-service: {{ .Release.Name }} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ tpl .Values.identifier . }}-letsencrypt-certs +spec: + accessModes: + - ReadWriteMany + resources: + requests: + storage: 50Mi + storageClassName: rook-cephfs +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ tpl .Values.identifier . }}-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: rook-ceph-block +# --- +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: {{ tpl .Values.identifier . }}-nginx-config +# data: +# default.conf: | +# server { +# listen 443 ssl; +# listen [::]:443 ssl; + +# server_name {{ tpl .Values.fqdn . }}; + +# ssl_certificate /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; +# ssl_certificate_key /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem; + +# client_max_body_size 256m; + +# location / { +# proxy_pass http://localhost:3000; +# } +# } diff --git a/apps/chartmuseum/templates/tests/test-connection.yaml b/apps/chartmuseum/templates/tests/test-connection.yaml new file mode 100644 index 0000000..85cbdda --- /dev/null +++ b/apps/chartmuseum/templates/tests/test-connection.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ tpl .Values.identifier . }}-test-connection" + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['https://{{ tpl .Values.fqdn . }}'] + restartPolicy: Never diff --git a/apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz b/apps/chartmuseum/ungleich-chartmuseum-0.1.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..d592a80b3cc083daf2878c93191fb1c399b447c9 GIT binary patch literal 1633 zcmV-n2A=sJiwG0|00000|0w_~VMtOiV@ORlOnEsqVl!4SWK%V1T2nbTPgYhoO;>Dc zVQyr3R8em|NM&qo0PI<9Z`(NX&ue~)`P(o0@LxUHvE4$8!(QP8ivo2MBt`B;u`p!@V!rt(>KRSZ&QMB)&!dR-0!XMVv zbnclD&G{8-!VV3DPgFaEg+6rZI;pfxC&Ugh@E(uB(-g3by}hY%j4w0zD{N6@*s z0sjI8QiJ|SnjzfYcF6pXShmmk%`uZxR!*2DD<`k2z}zvq6#7?hK*d`Dai9YQttt1o||CVBMUC$2u`-gPcpPLrj|wo zpsMUG8N5^EO^(d1HYku;rQJI~MlDM}&>0F-4S>_RUmAeMTyREB1U&+1~)fg zbFoES2ioAn?QJkGgkUM<($6uwNA3bw4;7nhzFgp{Ar$4Z>2I~Am!Hlq-u-&IGY3%i zEfI8tVpMd##W?+NHh%YEdU1C4>E!*zJz}ckFm*voC5j9us*qowM5O>Nt9jj!oH@pG zYEvh@u|e(jx}g4+kX;ii^IAgiUaNAK_{O5NS0(RQP@e62#@x`yP0H7g@2=RFhv?N* zl(A*LvZLY7qiQc@H}Px>WtUYpx8vA1#FAYw*#OD6I5xOoS2;#7u9}ti=cJy&Uin_n z*UFgG+PpqL7#ugJO%qeU2e0nkm-DtGcmJ{{nNVy&y5KzQLc_-{R9=piobzNtbJDF<#V&lOcQWDf+GC{<(uF(j80WMQ)^}48 z{JR`;aj2K5;b#5mNR)f*0vC)CKYu;*If{OZw$ruwO4!ud*)(%oD0Uf)n7Ux&bIz#6 zYVb$u+0XY2pDu9qVBxi@wqrOG9HpIR^m;l|@oH-Lp9uZoXm^z>W0ork!OjMkz^T>s zYMN7HN=S%;psabRGB!F6!?5ni?fU(;?UuBs|F{mNC4>yN+vfeUk>MQj1Omj%N#|%^rHwG<->L z)3U6a=2>A@Gj+Y~y|auuyFb5hOJDr9xmUQ>7e2ozi)Iwuo2Dg$Mi&V7TnWxr5q!iY z>c7Fy3a$C?yHnzcR2m;c$G-d@9)I=w|8Q`)|33+J;hb8FS{kqlsxyZ#DN2|XT*O=^ zkW;px2^ybt;Zw>Dm?AG5SuiOI0f|tv>RQib@)Kwj)bb@lPHkEnr!pp8keF08?PacU z&aW|sFWjc^f3JKvlVSx*mYmKccRc{XCHllaxtv~FrIB=Dtg=i=xEfzV%(WqY!fjAK zEAxpz`%?$yb2E@8fqQH|%~A$;f@YL0iX7%#pdqh)^Cc&*{TW@5*S^i%yV5)%um3~3 za78s&g@N}cr-t~sR=*>!OKs^wb!PW+`|D#qYBz(;HTU!1->nVkAr{5U%H{{L^# z%>Th?G&tn{Q&6`H6K)Kb2_YWDGcd?BT1Zf$UT3ONx%uT4@!)du=^PAN%VlC9Sg$qk za{Dj9jB)}04_l7Y>s&eE;5u9bTvk7>lpQ3zjAuVV!Y$HQ#8vwVUCj_Q#szaMRzVNW fug1SThdesafev(_=SKex00960*|9nL03-kadSN%p literal 0 HcmV?d00001 diff --git a/apps/chartmuseum/values.yaml b/apps/chartmuseum/values.yaml new file mode 100644 index 0000000..bdf3229 --- /dev/null +++ b/apps/chartmuseum/values.yaml @@ -0,0 +1,6 @@ +clusterDomain: c2.k8s.ooo +email: technik@ungleich.ch +letsencryptStaging: "yes" + +identifier: "{{ .Release.Name }}" +fqdn: "{{ .Release.Name }}.{{ .Release.Namespace }}.svc.{{ .Values.clusterDomain }}" From 90396d973b4f264b20a26ff2a3aa96efb35e8d71 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Jul 2021 00:00:37 +0200 Subject: [PATCH 2/6] [knotdns] use Appversion in chart --- apps/knotdns/Chart.yaml | 5 +++-- apps/knotdns/templates/deployment.yaml | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/knotdns/Chart.yaml b/apps/knotdns/Chart.yaml index d2d962b..c1d5334 100644 --- a/apps/knotdns/Chart.yaml +++ b/apps/knotdns/Chart.yaml @@ -1,5 +1,6 @@ apiVersion: v2 name: knotdns -description: DNS Auth +description: Authoritative DNS server -version: 0.1.0 +version: 0.1.1 +appVersion: 3.0 diff --git a/apps/knotdns/templates/deployment.yaml b/apps/knotdns/templates/deployment.yaml index 2eeaf29..613b0a2 100644 --- a/apps/knotdns/templates/deployment.yaml +++ b/apps/knotdns/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: knot - image: cznic/knot:3.0 + image: cznic/knot:{{ .Chart.AppVersion }} ports: - name: udp-53 containerPort: 53 From ae57db29360b7ec82e81917c2c9b0ad931cd7630 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Jul 2021 00:00:46 +0200 Subject: [PATCH 3/6] ++readme --- README.md | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5acb5f3..4367b78 100644 --- a/README.md +++ b/README.md @@ -13,13 +13,14 @@ This project is testing, deploying and using IPv6 only k8s clusters. * ceph with rook (cephfs, rbd) * letsencrypt (nginx, certbot, homemade) * k8s test on arm64 +* CI/CD using flux +* Chart repository (chartmuseum) ## Not (yet) working or tested * virtualisation (VMs, kubevirt) * network policies -* prometheus in the cluster -* -argocd (?) for CI and upgrades- using flux +* Prometheus for the cluster * Maybe LoadBalancer support (our ClusterIP already does that though) * (Other) DNS entrys for services * Internal backup / snapshots @@ -242,6 +243,8 @@ referes to an external load balancer that ## Secrets +### Generating them inside the cluster + Handled via https://github.com/mittwald/kubernetes-secret-generator ``` @@ -263,7 +266,36 @@ data: username: c29tZXVzZXI= ``` -This will add a password into it. Password only will +* Advantage: passwords are only in the cluster +* Disadvantage: passwords are only in the cluster + +## CI/CD + +### What we want + +* Package everything into one git repository (charts, kustomize, etc.) +* Be usable for multiple clusters +* Easily apply cross cluster + +### What we don't want / what is problematic + +* Uploading charts to something like chartmuseum + * Is redundant - we have a version in git + * Is manual (could probably be automated) + +### ArgoCD + +Looks too big, too complex, too complicated. + +### FluxCD2 + +Looks ok, handling of helm is ok, but does not feel intuitive. Seems +to be more orientated on "kustomizing helm charts". + +### Helmfile + +[helmfile](https://github.com/roboll/helmfile/) seems to do most of +what we need. ## The IPv4 "problem" From da95831382a7aabdc0f623217a50089a0918e70e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Jul 2021 13:12:36 +0200 Subject: [PATCH 4/6] ++docs --- README.md | 4 ++++ apps/chartmuseum/README.md | 17 +++++++++++++---- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4367b78..5cbb353 100644 --- a/README.md +++ b/README.md @@ -15,9 +15,13 @@ This project is testing, deploying and using IPv6 only k8s clusters. * k8s test on arm64 * CI/CD using flux * Chart repository (chartmuseum) +* Git repository (gitea) ## Not (yet) working or tested +* proxy for pulling images only + * configure a proxy on crio + * setup a proxy in the cluster (?) * virtualisation (VMs, kubevirt) * network policies * Prometheus for the cluster diff --git a/apps/chartmuseum/README.md b/apps/chartmuseum/README.md index 7d68f32..080001d 100644 --- a/apps/chartmuseum/README.md +++ b/apps/chartmuseum/README.md @@ -1,6 +1,15 @@ ## Missing -- db secret generation (sops?) -- SMTP settings / secrets (ungleich mail + sops?) -- Exposing sizes in value.yaml (db, gitea) - - Maybe reducing to 1 PVC? +* SSL/TLS + * Letsencrypt on port 80 + * Chartmuseum on 443 + * Reload? +* Authentication + * Secret -> generate + * Secret -> sops + + +## Done + +* Basic chartmuseum +* helm push works From 88d7f27247a1744e78b11f63aaaae7f25dc1180f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Jul 2021 13:50:28 +0200 Subject: [PATCH 5/6] chartmuseum: phase in SSL support --- apps/chartmuseum/Chart.yaml | 2 +- apps/chartmuseum/templates/deployment.yaml | 83 ++++++++++++++++++++-- 2 files changed, 77 insertions(+), 8 deletions(-) diff --git a/apps/chartmuseum/Chart.yaml b/apps/chartmuseum/Chart.yaml index 4eef696..84e2f11 100644 --- a/apps/chartmuseum/Chart.yaml +++ b/apps/chartmuseum/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: ungleich-chartmuseum description: Chartmuseum for the ungleich infrastructure -version: 0.1.0 +version: 0.1.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/apps/chartmuseum/templates/deployment.yaml b/apps/chartmuseum/templates/deployment.yaml index 592deee..f811498 100644 --- a/apps/chartmuseum/templates/deployment.yaml +++ b/apps/chartmuseum/templates/deployment.yaml @@ -14,14 +14,44 @@ spec: app: {{ .Release.Name }}-chartmuseum use-as-service: {{ .Release.Name }} spec: + # Wait before trying to start any container + initcontainers: + - name: wait-for-cert + image: busybox + command: + - sh + - -c + - until ls /etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem; do sleep 5; done + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" containers: - - name: chartmuseum - image: ghcr.io/helm/chartmuseum:v0.13.1 + - name: certbot + image: ungleich/ungleich-certbot + imagePullPolicy: Always ports: - - containerPort: 8080 - # args: - # - --tls-cert=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem - # - --tls-key=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem + - containerPort: 80 + env: + - name: ONLYRENEWCERTS + value: "yes" + - name: DOMAIN + value: "{{ tpl .Values.fqdn . }}" + - name: EMAIL + value: "{{ .Values.email }}" + {{ if eq .Values.letsencryptStaging "no" }} + - name: STAGING + value: "no" + {{ end }} + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + - name: chartmuseum + image: ghcr.io/helm/chartmuseum:v{{ .Chart.AppVersion }} + ports: + - containerPort: 443 + args: + - --tls-cert=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/fullchain.pem + - --tls-key=/etc/letsencrypt/live/{{ tpl .Values.fqdn . }}/privkey.pem env: - name: STORAGE value: "local" @@ -49,7 +79,7 @@ metadata: spec: type: ClusterIP ports: - - port: 8080 + - port: 80 name: http - port: 443 name: https @@ -101,3 +131,42 @@ spec: # proxy_pass http://localhost:3000; # } # } +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ tpl .Values.identifier . }}-getcert +spec: + template: + metadata: + labels: + app: certbot-letsencrypt-getcert + use-as-service: {{ .Release.Name }} + spec: + restartPolicy: Never + containers: + - name: certbot + image: ungleich/ungleich-certbot + imagePullPolicy: Always + + ports: + - containerPort: 80 + env: + - name: ONLYGETCERT + value: "yes" + - name: DOMAIN + value: "{{ tpl .Values.fqdn . }}" + - name: EMAIL + value: "{{ .Values.email }}" + {{ if eq .Values.letsencryptStaging "no" }} + - name: STAGING + value: "no" + {{ end }} + volumeMounts: + - name: etcletsencrypt + mountPath: "/etc/letsencrypt" + volumes: + - name: etcletsencrypt + persistentVolumeClaim: + claimName: {{ tpl .Values.identifier . }}-letsencrypt-certs + backoffLimit: 3 From 4dc1bb194c9550eed18b511b122db6390e02200b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Jul 2021 14:09:48 +0200 Subject: [PATCH 6/6] knot: quote to make 3->3.0 --- apps/knotdns/Chart.yaml | 2 +- apps/knotdns/templates/deployment.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/knotdns/Chart.yaml b/apps/knotdns/Chart.yaml index c1d5334..907aefa 100644 --- a/apps/knotdns/Chart.yaml +++ b/apps/knotdns/Chart.yaml @@ -3,4 +3,4 @@ name: knotdns description: Authoritative DNS server version: 0.1.1 -appVersion: 3.0 +appVersion: "3.0" diff --git a/apps/knotdns/templates/deployment.yaml b/apps/knotdns/templates/deployment.yaml index 613b0a2..2c20348 100644 --- a/apps/knotdns/templates/deployment.yaml +++ b/apps/knotdns/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: spec: containers: - name: knot - image: cznic/knot:{{ .Chart.AppVersion }} + image: "cznic/knot:{{ .Chart.AppVersion }}" ports: - name: udp-53 containerPort: 53