From 092eebe3cfe60be02207ee7cea39f4ba94f95dee Mon Sep 17 00:00:00 2001
From: xfk <fran.kristo42@gmail.com>
Date: Thu, 29 Jul 2021 18:20:39 +0200
Subject: [PATCH] ++c4

---
 k8s/c4/README.md    | 56 +++++++++++++++++++++++++++++++++++++++++++++
 k8s/c4/bgp-c4.yaml  | 21 +++++++++++++++++
 k8s/c4/kubeadm.yaml | 21 +++++++++++++++++
 3 files changed, 98 insertions(+)
 create mode 100644 k8s/c4/README.md
 create mode 100644 k8s/c4/bgp-c4.yaml
 create mode 100644 k8s/c4/kubeadm.yaml

diff --git a/k8s/c4/README.md b/k8s/c4/README.md
new file mode 100644
index 0000000..c1410a3
--- /dev/null
+++ b/k8s/c4/README.md
@@ -0,0 +1,56 @@
+## c4.k8s.ooo
+HA Cluster: 3 Control-plane nodes + 4 Worker nodes
+
+Note: files in commands are referenced from the repo root directory
+
+## controlPlaneEndpoint loadbalancer
+Configured via HAProxy on router1
+c4-api.k8s.ooo:6443 -> server52, server53, server54
+
+## Initialise 1st control-plane node
+```
+kubeadm init --config k8s/c4/kubeadm.yaml --upload-certs
+```
+
+## Add CNI
+```
+kubectl apply -f cni-calico/calico.yaml
+```
+
+## Join control-plane node 2 and 3
+```
+kubeadm join c4-api.k8s.ooo:6443 --token *.* \
+--discovery-token-ca-cert-hash sha256:* --control-plane \
+--certificate-key *** --cri-socket=/var/run/crio/crio.sock
+```
+
+## Join worker nodes
+```
+kubeadm join c4-api.k8s.ooo:6443 --token *.* \
+--discovery-token-ca-cert-hash sha256:* \
+--cri-socket=/var/run/crio/crio.sock
+```
+
+## Configure BGP
+```
+kubectl apply -f https://docs.projectcalico.org/manifests/calicoctl.yaml
+alias calicoctl="kubectl exec -i -n kube-system calicoctl -- /calicoctl"
+calicoctl create -f - < k8s/c4/bgp-c4.yaml
+```
+
+## Rook
+```
+for yaml in crds common operator cluster storageclass-cephfs storageclass-rbd toolbox; do
+    kubectl apply -f rook/${yaml}.yaml
+done
+```
+
+Set Rook as default storage class:
+```
+kubectl patch storageclass rook-ceph-block -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
+```
+
+Get Rook admin password:
+```
+kubectl -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo
+```
diff --git a/k8s/c4/bgp-c4.yaml b/k8s/c4/bgp-c4.yaml
new file mode 100644
index 0000000..cc9ee0a
--- /dev/null
+++ b/k8s/c4/bgp-c4.yaml
@@ -0,0 +1,21 @@
+---
+apiVersion: projectcalico.org/v3
+kind: BGPConfiguration
+metadata:
+  name: default
+spec:
+  logSeverityScreen: Info
+  nodeToNodeMeshEnabled: true
+  asNumber: 65534
+  serviceClusterIPs:
+  - cidr: 2a0a:e5c0:13:f2::/108
+  serviceExternalIPs:
+  - cidr: 2a0a:e5c0:13:f2::/108
+---
+apiVersion: projectcalico.org/v3
+kind: BGPPeer
+metadata:
+  name: red-place7
+spec:
+  peerIP: 2a0a:e5c0:13::42
+  asNumber: 213081
diff --git a/k8s/c4/kubeadm.yaml b/k8s/c4/kubeadm.yaml
new file mode 100644
index 0000000..a9bbef9
--- /dev/null
+++ b/k8s/c4/kubeadm.yaml
@@ -0,0 +1,21 @@
+kind: InitConfiguration
+apiVersion: kubeadm.k8s.io/v1beta2
+localAPIEndpoint:
+  bindPort: 6443
+nodeRegistration:
+  criSocket: "unix:///var/run/crio/crio.sock"
+  kubeletExtraArgs:
+    cgroup-driver: "cgroupfs"
+---
+kind: ClusterConfiguration
+apiVersion: kubeadm.k8s.io/v1beta2
+kubernetesVersion: v1.21.1
+networking:
+  dnsDomain: c4.k8s.ooo
+  podSubnet: 2a0a:e5c0:13:f1::/64
+  serviceSubnet: 2a0a:e5c0:13:f2::/108
+controlPlaneEndpoint: "c4-api.k8s.ooo:6443"
+---
+kind: KubeletConfiguration
+apiVersion: kubelet.config.k8s.io/v1beta1
+cgroupDriver: cgroupfs