diff --git a/README.md b/README.md index df0c467..7d75058 100644 --- a/README.md +++ b/README.md @@ -21,3 +21,19 @@ This project is testing, deploying and using IPv6 only k8s clusters. * argocd (?) for CI and upgrades * Maybe LoadBalancer support (our ClusterIP already does that though) * (Other) DNS entrys for services + +## Cluster setup + +* Calico CNI with BGP peering to our upstream infrastructure +* Rook for RBD and CephFS support + +### Init + +We are using a custom kubeadm.conf to + +* configure the cgroupdriver +* configure the IP addresses + +``` +kubeadm init --config v3-calico/kubeadm-config-p7-v2.yaml +``` diff --git a/certificates/certbot-homemade.yaml b/certificates/certbot-homemade.yaml index 9da0c13..4f9d3b2 100644 --- a/certificates/certbot-homemade.yaml +++ b/certificates/certbot-homemade.yaml @@ -50,17 +50,18 @@ spec: apiVersion: v1 kind: Pod metadata: - name: nginx-port-80 - labels: - app: nginx-port-80 + name: alpine-cert-test spec: containers: - - name: nginx - image: nginx:1.21-alpine + - name: alpine + image: alpine:3.13 + args: + - sleep + - "1000000" volumeMounts: - mountPath: "/etc/letsencrypt" name: letsencryptdir - - mountPath: "/usr/share/nginx/html" + - mountPath: "/www" name: webroot volumes: - name: letsencryptdir @@ -71,6 +72,57 @@ spec: claimName: nginx-webroot --- apiVersion: v1 +kind: Pod +metadata: + name: nginx-port-80 + labels: + app: ssl +spec: + containers: + - name: nginx + image: nginx:1.21-alpine + volumeMounts: + - mountPath: "/etc/letsencrypt" + name: letsencryptdir + - mountPath: "/usr/share/nginx/html" + name: webroot + ports: + - containerPort: 80 + volumes: + - name: letsencryptdir + persistentVolumeClaim: + claimName: nginx-certs + - name: webroot + persistentVolumeClaim: + claimName: nginx-webroot +# --- +# apiVersion: v1 +# kind: Pod +# metadata: +# name: nginx-ssl +# labels: +# app: nginx-ssl +# ssl: yes +# spec: +# containers: +# - name: nginx +# image: nginx:1.21-alpine +# volumeMounts: +# - mountPath: "/etc/letsencrypt" +# name: letsencryptdir +# - mountPath: "/usr/share/nginx/html" +# name: webroot +# ports: +# - containerPort: 443 +# volumes: +# - name: letsencryptdir +# persistentVolumeClaim: +# claimName: nginx-certs +# - name: webroot +# persistentVolumeClaim: +# claimName: nginx-webroot +--- +apiVersion: v1 kind: Service metadata: name: svc1 @@ -78,5 +130,25 @@ spec: type: ClusterIP ports: - port: 80 + name: "http" + - port: 443 + name: "https" selector: - app: nginx-port-80 + app: ssl +# --- +# apiVersion: v1 +# kind: ConfigMap +# metadata: +# name: nginx-ssl-config +# data: +# # property-like keys; each key maps to a simple value +# nick_name: "13" + +# # file-like keys +# game.properties: | +# enemy.types=aliens,monsters +# player.maximum-lives=5 +# user-interface.properties: | +# color.good=purple +# color.bad=yellow +# allow.textmode=true diff --git a/rook/README.md b/rook/README.md index c87d6bc..8565ffc 100644 --- a/rook/README.md +++ b/rook/README.md @@ -22,6 +22,15 @@ for yaml in crds common operator cluster storageclass toolbox; do done ``` +Deleting (in case of teardown): + +``` +for yaml in crds common operator cluster storageclass toolbox; do + kubectl delete -f ${yaml}.yaml +done +``` + + ## Debugging / ceph toolbox ``` diff --git a/rook/osd-purge.yaml b/rook/osd-purge.yaml new file mode 100644 index 0000000..3725b45 --- /dev/null +++ b/rook/osd-purge.yaml @@ -0,0 +1,73 @@ +################################################################################################################# +# We need many operations to remove OSDs as written in Documentation/ceph-osd-mgmt.md. +# This job can automate some of that operations: mark OSDs as `out`, purge these OSDs, +# and delete the corresponding resources like OSD deployments, OSD prepare jobs, and PVCs. +# +# Please note the following. +# +# - This job only works for `down` OSDs. +# - This job doesn't wait for backfilling to be completed. +# +# If you want to remove `up` OSDs and/or want to wait for backfilling to be completed between each OSD removal, +# please do it by hand. +################################################################################################################# + +apiVersion: batch/v1 +kind: Job +metadata: + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:operator + labels: + app: rook-ceph-purge-osd +spec: + template: + spec: + serviceAccountName: rook-ceph-system + containers: + - name: osd-removal + image: rook/ceph:master + # TODO: Insert the OSD ID in the last parameter that is to be removed + # The OSD IDs are a comma-separated list. For example: "0" or "0,2". + args: ["ceph", "osd", "remove", "--osd-ids", "3,1,4,2,5"] + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ROOK_MON_ENDPOINTS + valueFrom: + configMapKeyRef: + key: data + name: rook-ceph-mon-endpoints + - name: ROOK_CEPH_USERNAME + valueFrom: + secretKeyRef: + key: ceph-username + name: rook-ceph-mon + - name: ROOK_CEPH_SECRET + valueFrom: + secretKeyRef: + key: ceph-secret + name: rook-ceph-mon + - name: ROOK_CONFIG_DIR + value: /var/lib/rook + - name: ROOK_CEPH_CONFIG_OVERRIDE + value: /etc/rook/config/override.conf + - name: ROOK_FSID + valueFrom: + secretKeyRef: + key: fsid + name: rook-ceph-mon + - name: ROOK_LOG_LEVEL + value: DEBUG + volumeMounts: + - mountPath: /etc/ceph + name: ceph-conf-emptydir + - mountPath: /var/lib/rook + name: rook-config + volumes: + - emptyDir: {} + name: ceph-conf-emptydir + - emptyDir: {} + name: rook-config + restartPolicy: Never diff --git a/v3-calico/kubeadm-config-p7-v2.yaml b/v3-calico/kubeadm-config-p7-v2.yaml index 05fcb63..4671591 100644 --- a/v3-calico/kubeadm-config-p7-v2.yaml +++ b/v3-calico/kubeadm-config-p7-v2.yaml @@ -1,7 +1,7 @@ # kubeadm-config.yaml kind: ClusterConfiguration apiVersion: kubeadm.k8s.io/v1beta2 -kubernetesVersion: v1.21.0 +kubernetesVersion: v1.21.1 networking: dnsDomain: k8s.place7.ungleich.ch podSubnet: 2a0a:e5c0:13:e1::/64