diff --git a/k8s/p6/calico-bgp.yaml b/k8s/p6/calico-bgp.yaml new file mode 100644 index 0000000..a037db9 --- /dev/null +++ b/k8s/p6/calico-bgp.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: projectcalico.org/v3 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: true + asNumber: 65534 + serviceClusterIPs: + - cidr: 2a0a:e5c0:2:14::/108 + serviceExternalIPs: + - cidr: 2a0a:e5c0:2:14::/108 +--- +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: apu-router1-place6 +spec: + peerIP: 2a0a:e5c0:2:1::43 + asNumber: 207996 +--- +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: apu-router2-place6 +spec: + peerIP: 2a0a:e5c0:2:1::44 + asNumber: 207996 diff --git a/k8s/p6/kubeadm.yaml b/k8s/p6/kubeadm.yaml new file mode 100644 index 0000000..76099fd --- /dev/null +++ b/k8s/p6/kubeadm.yaml @@ -0,0 +1,13 @@ +# kubeadm-config.yaml +kind: ClusterConfiguration +apiVersion: kubeadm.k8s.io/v1beta2 +kubernetesVersion: v1.21.2 +networking: + dnsDomain: p6.k8s.ooo + podSubnet: 2a0a:e5c0:2:13::/64 + serviceSubnet: 2a0a:e5c0:2:14::/108 +controlPlaneEndpoint: "p6-api.k8s.ooo:6443" +--- +kind: KubeletConfiguration +apiVersion: kubelet.config.k8s.io/v1beta1 +cgroupDriver: cgroupfs diff --git a/v3-calico/calico.yaml b/v3-calico/calico.yaml index 4738b67..492da50 100644 --- a/v3-calico/calico.yaml +++ b/v3-calico/calico.yaml @@ -32,9 +32,9 @@ data: "nodename": "__KUBERNETES_NODE_NAME__", "mtu": __CNI_MTU__, "ipam": { - "type": "calico-ipam", "assign_ipv4": "false", - "assign_ipv6": "true" + "assign_ipv6": true", + "type": "calico-ipam" }, "policy": { "type": "k8s" @@ -243,6 +243,11 @@ spec: Peers node to use the "next hop keep;" instead of "next hop self;"(default) in the specific branch of the Node on "bird.cfg". type: boolean + maxRestartTime: + description: Time to allow for software restart. When specified, this + is configured as the graceful restart timeout. When not specified, + the BIRD default of 120s is used. + type: string node: description: The node name identifying the Calico node instance that is targeted by this peer. If this is not set, and no nodeSelector @@ -513,13 +518,6 @@ spec: description: 'BPFEnabled, if enabled Felix will use the BPF dataplane. [Default: false]' type: boolean - bpfExtToServiceConnmark: - description: 'BPFExtToServiceConnmark in BPF mode, control a 32bit - mark that is set on connections from an external client to a local - service. This mark allows us to control how packets of that connection - are routed within the host and how is routing intepreted by RPF - check. [Default: 0]' - type: integer bpfExternalServiceMode: description: 'BPFExternalServiceMode in BPF mode, controls how connections from outside the cluster to services (node ports and cluster IPs) @@ -530,6 +528,14 @@ spec: node appears to use the IP of the ingress node; this requires a permissive L2 network. [Default: Tunnel]' type: string + bpfExtToServiceConnmark: + description: 'BPFExtToServiceConnmark in BPF mode, controls a + 32bit mark that is set on connections from an external client to + a local service. This mark allows us to control how packets of + that connection are routed within the host and how is routing + intepreted by RPF check. [Default: 0]' + type: integer + bpfKubeProxyEndpointSlicesEnabled: description: BPFKubeProxyEndpointSlicesEnabled in BPF mode, controls whether Felix's embedded kube-proxy accepts EndpointSlices or not. @@ -1063,16 +1069,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1160,6 +1167,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1268,16 +1295,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1365,6 +1393,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -1394,16 +1442,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1491,6 +1540,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -1599,16 +1668,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -1696,6 +1766,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -2280,6 +2370,11 @@ spec: host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the controller + to determine if an IP address has been leaked. Set to 0 + to disable IP garbage collection. [Default: 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: 5m]' @@ -2380,6 +2475,12 @@ spec: of host endpoints for every node. [Default: Disabled]' type: string type: object + leakGracePeriod: + description: 'LeakGracePeriod is the period used by the + controller to determine if an IP address has been leaked. + Set to 0 to disable IP garbage collection. [Default: + 15m]' + type: string reconcilerPeriod: description: 'ReconcilerPeriod is the period to perform reconciliation with the Calico datastore. [Default: @@ -2508,16 +2609,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2605,6 +2707,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -2713,16 +2835,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2810,6 +2933,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -2839,16 +2982,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -2936,6 +3080,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object http: description: HTTP contains match criteria that apply to HTTP @@ -3044,16 +3208,17 @@ spec: contains a selector expression. Only traffic that originates from (or terminates at) endpoints within the selected namespaces will be matched. When both NamespaceSelector - and Selector are defined on the same rule, then only workload - endpoints that are matched by both selectors will be selected - by the rule. \n For NetworkPolicy, an empty NamespaceSelector - implies that the Selector is limited to selecting only - workload endpoints in the same namespace as the NetworkPolicy. - \n For NetworkPolicy, `global()` NamespaceSelector implies - that the Selector is limited to selecting only GlobalNetworkSet - or HostEndpoint. \n For GlobalNetworkPolicy, an empty - NamespaceSelector implies the Selector applies to workload - endpoints across all namespaces." + and another selector are defined on the same rule, then + only workload endpoints that are matched by both selectors + will be selected by the rule. \n For NetworkPolicy, an + empty NamespaceSelector implies that the Selector is limited + to selecting only workload endpoints in the same namespace + as the NetworkPolicy. \n For NetworkPolicy, `global()` + NamespaceSelector implies that the Selector is limited + to selecting only GlobalNetworkSet or HostEndpoint. \n + For GlobalNetworkPolicy, an empty NamespaceSelector implies + the Selector applies to workload endpoints across all + namespaces." type: string nets: description: Nets is an optional field that restricts the @@ -3141,6 +3306,26 @@ spec: AND'ed. type: string type: object + services: + description: "Services is an optional field that contains + options for matching Kubernetes Services. If specified, + only traffic that originates from or terminates at endpoints + within the selected service(s) will be matched, and only + to/from each endpoint's port. \n Services cannot be specified + on the same rule as Selector, NotSelector, NamespaceSelector, + Ports, NotPorts, Nets, NotNets or ServiceAccounts. \n + Only valid on egress rules." + properties: + name: + description: Name specifies the name of a Kubernetes + Service to match. + type: string + namespace: + description: Namespace specifies the namespace of the + given Service. If left empty, the rule will match + within this policy's namespace. + type: string + type: object type: object required: - action @@ -3274,12 +3459,14 @@ rules: - watch - list - get - # Pods are queried to check for existence. + # Pods are watched to check for existence as part of IPAM controller. - apiGroups: [""] resources: - pods verbs: - get + - list + - watch # IPAM resources are manipulated when nodes are deleted. - apiGroups: ["crd.projectcalico.org"] resources: @@ -3361,6 +3548,14 @@ rules: - namespaces verbs: - get + # EndpointSlices are used for Service-based network policy rule + # enforcement. + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - watch + - list - apiGroups: [""] resources: - endpoints @@ -3546,7 +3741,7 @@ spec: # It can be deleted if this is a fresh installation, or if you have already # upgraded to use calico-ipam. - name: upgrade-ipam - image: docker.io/calico/cni:v3.19.0 + image: docker.io/calico/cni:v3.20.0 command: ["/opt/cni/bin/calico-ipam", "-upgrade"] envFrom: - configMapRef: @@ -3573,7 +3768,7 @@ spec: # This container installs the CNI binaries # and CNI network config file on each node. - name: install-cni - image: docker.io/calico/cni:v3.19.0 + image: docker.io/calico/cni:v3.20.0 command: ["/opt/cni/bin/install"] envFrom: - configMapRef: @@ -3614,7 +3809,7 @@ spec: # Adds a Flex Volume Driver that creates a per-pod Unix Domain Socket to allow Dikastes # to communicate with Felix over the Policy Sync API. - name: flexvol-driver - image: docker.io/calico/pod2daemon-flexvol:v3.19.0 + image: docker.io/calico/pod2daemon-flexvol:v3.20.0 volumeMounts: - name: flexvol-driver-host mountPath: /host/driver @@ -3625,7 +3820,7 @@ spec: # container programs network policy and routes on each # host. - name: calico-node - image: docker.io/calico/node:v3.19.0 + image: docker.io/calico/node:v3.20.0 envFrom: - configMapRef: # Allow KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT to be overridden for eBPF mode. @@ -3713,6 +3908,7 @@ spec: periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: exec: command: @@ -3720,7 +3916,12 @@ spec: - -felix-ready - -bird-ready periodSeconds: 10 + timeoutSeconds: 10 volumeMounts: + # For maintaining CNI plugin API credentials. + - mountPath: /host/etc/cni/net.d + name: cni-net-dir + readOnly: false - mountPath: /lib/modules name: lib-modules readOnly: true @@ -3836,7 +4037,7 @@ spec: priorityClassName: system-cluster-critical containers: - name: calico-kube-controllers - image: docker.io/calico/kube-controllers:v3.19.0 + image: docker.io/calico/kube-controllers:v3.20.0 env: # Choose which controllers to run. - name: ENABLED_CONTROLLERS @@ -3851,6 +4052,7 @@ spec: periodSeconds: 10 initialDelaySeconds: 10 failureThreshold: 6 + timeoutSeconds: 10 readinessProbe: exec: command: