From 4058ff5287545cf2f4fb6a6b3c4e77f4031b2d3a Mon Sep 17 00:00:00 2001 From: Dreaded_X Date: Wed, 26 Feb 2025 00:30:26 +0100 Subject: [PATCH] Use kyverno to add annotation to kube-vip pods --- .../infra/kube-vip.yaml | 2 + .../infra/kyverno-policies.yaml | 16 ++++ infra/kube-vip/daemon-set-enp3s0.yaml | 88 ------------------- ...daemon-set-enp2s0.yaml => daemon-set.yaml} | 17 ++-- infra/kube-vip/kustomization.yaml | 3 +- .../kube-vip-network-adapter.yaml | 37 ++++++++ infra/kyverno-policies/kustomization.yaml | 4 + infra/kyverno/values.yaml | 19 ++++ 8 files changed, 89 insertions(+), 97 deletions(-) create mode 100644 clusters/titan.lan.huizinga.dev/infra/kyverno-policies.yaml delete mode 100644 infra/kube-vip/daemon-set-enp3s0.yaml rename infra/kube-vip/{daemon-set-enp2s0.yaml => daemon-set.yaml} (84%) create mode 100644 infra/kyverno-policies/kube-vip-network-adapter.yaml create mode 100644 infra/kyverno-policies/kustomization.yaml diff --git a/clusters/titan.lan.huizinga.dev/infra/kube-vip.yaml b/clusters/titan.lan.huizinga.dev/infra/kube-vip.yaml index 9f6d74f..df80c86 100644 --- a/clusters/titan.lan.huizinga.dev/infra/kube-vip.yaml +++ b/clusters/titan.lan.huizinga.dev/infra/kube-vip.yaml @@ -6,6 +6,8 @@ metadata: spec: interval: 15m path: ./infra/kube-vip + dependsOn: + - name: kyverno-policies prune: true timeout: 2m sourceRef: diff --git a/clusters/titan.lan.huizinga.dev/infra/kyverno-policies.yaml b/clusters/titan.lan.huizinga.dev/infra/kyverno-policies.yaml new file mode 100644 index 0000000..e28c290 --- /dev/null +++ b/clusters/titan.lan.huizinga.dev/infra/kyverno-policies.yaml @@ -0,0 +1,16 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: kyverno-policies + namespace: flux-system +spec: + interval: 15m + path: ./infra/kyverno-policies + dependsOn: + - name: kyverno + prune: true + timeout: 2m + sourceRef: + kind: GitRepository + name: flux-system + wait: true diff --git a/infra/kube-vip/daemon-set-enp3s0.yaml b/infra/kube-vip/daemon-set-enp3s0.yaml deleted file mode 100644 index 3f92076..0000000 --- a/infra/kube-vip/daemon-set-enp3s0.yaml +++ /dev/null @@ -1,88 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.8.3 - name: kube-vip-ds-enp3s0 -spec: - selector: - matchLabels: - app.kubernetes.io/name: kube-vip-ds - template: - metadata: - labels: - app.kubernetes.io/name: kube-vip-ds - app.kubernetes.io/version: v0.8.3 - spec: - affinity: - nodeAffinity: - requiredDuringSchedulingIgnoredDuringExecution: - nodeSelectorTerms: - - matchExpressions: - - key: node-role.kubernetes.io/master - operator: Exists - - matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists - nodeSelector: - feature.node.kubernetes.io/network-adapter: enp3s0 - containers: - - args: - - manager - env: - - name: vip_arp - value: "true" - - name: port - value: "6443" - - name: vip_nodename - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: vip_interface - value: enp3s0 - - name: vip_cidr - value: "32" - - name: dns_mode - value: first - - name: cp_enable - value: "true" - - name: cp_namespace - value: kube-system - - name: svc_enable - value: "true" - - name: svc_election - value: "true" - - name: svc_leasename - value: plndr-svcs-lock - - name: vip_leaderelection - value: "true" - - name: vip_leasename - value: plndr-cp-lock - - name: vip_leaseduration - value: "5" - - name: vip_renewdeadline - value: "3" - - name: vip_retryperiod - value: "1" - - name: address - value: 10.0.2.1 - - name: prometheus_server - value: :2112 - image: ghcr.io/kube-vip/kube-vip:v0.8.3 - imagePullPolicy: IfNotPresent - name: kube-vip - resources: {} - securityContext: - capabilities: - add: - - NET_ADMIN - - NET_RAW - hostNetwork: true - serviceAccountName: kube-vip - tolerations: - - effect: NoSchedule - operator: Exists - - effect: NoExecute - operator: Exists - updateStrategy: {} diff --git a/infra/kube-vip/daemon-set-enp2s0.yaml b/infra/kube-vip/daemon-set.yaml similarity index 84% rename from infra/kube-vip/daemon-set-enp2s0.yaml rename to infra/kube-vip/daemon-set.yaml index 76d0b1b..cc59b97 100644 --- a/infra/kube-vip/daemon-set-enp2s0.yaml +++ b/infra/kube-vip/daemon-set.yaml @@ -2,17 +2,17 @@ apiVersion: apps/v1 kind: DaemonSet metadata: labels: - app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/name: kube-vip app.kubernetes.io/version: v0.8.3 - name: kube-vip-ds-enp2s0 + name: kube-vip spec: selector: matchLabels: - app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/name: kube-vip template: metadata: labels: - app.kubernetes.io/name: kube-vip-ds + app.kubernetes.io/name: kube-vip app.kubernetes.io/version: v0.8.3 spec: affinity: @@ -25,8 +25,9 @@ spec: - matchExpressions: - key: node-role.kubernetes.io/control-plane operator: Exists - nodeSelector: - feature.node.kubernetes.io/network-adapter: enp2s0 + - matchExpressions: + - key: feature.node.kubernetes.io/network-adapter + operator: Exists containers: - args: - manager @@ -40,7 +41,9 @@ spec: fieldRef: fieldPath: spec.nodeName - name: vip_interface - value: enp2s0 + valueFrom: + fieldRef: + fieldPath: metadata.annotations['feature.node.kubernetes.io/network-adapter'] - name: vip_cidr value: "32" - name: dns_mode diff --git a/infra/kube-vip/kustomization.yaml b/infra/kube-vip/kustomization.yaml index 9f63cf3..eee15f5 100644 --- a/infra/kube-vip/kustomization.yaml +++ b/infra/kube-vip/kustomization.yaml @@ -5,8 +5,7 @@ resources: - ./service-account.yaml - ./cluster-role.yaml - ./cluster-role-binding.yaml - - ./daemon-set-enp2s0.yaml - - ./daemon-set-enp3s0.yaml + - ./daemon-set.yaml - https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/refs/tags/v0.0.11/manifest/kube-vip-cloud-controller.yaml - ./config-map-kubevip.yaml diff --git a/infra/kyverno-policies/kube-vip-network-adapter.yaml b/infra/kyverno-policies/kube-vip-network-adapter.yaml new file mode 100644 index 0000000..474a2ac --- /dev/null +++ b/infra/kyverno-policies/kube-vip-network-adapter.yaml @@ -0,0 +1,37 @@ +apiVersion: kyverno.io/v2beta1 +kind: ClusterPolicy +metadata: + name: kube-vip-network-adapter + annotations: + pod-policies.kyverno.io/autogen-controllers: none + policies.kyverno.io/title: Kube VIP adapter label + policies.kyverno.io/category: Other + policies.kyverno.io/subject: Pod + kyverno.io/kyverno-version: 1.10.0 + policies.kyverno.io/minversion: 1.10.0 + kyverno.io/kubernetes-version: "1.26" +spec: + background: false + rules: + - name: add-network-adapter-annotation + match: + any: + - resources: + kinds: + - Pod/binding + names: + - kube-vip-* + context: + - name: node + variable: + jmesPath: request.object.target.name + default: "" + - name: adapter + apiCall: + urlPath: "/api/v1/nodes/{{node}}" + jmesPath: 'metadata.labels."feature.node.kubernetes.io/network-adapter" || "empty"' + mutate: + patchStrategicMerge: + metadata: + annotations: + feature.node.kubernetes.io/network-adapter: "{{ adapter }}" diff --git a/infra/kyverno-policies/kustomization.yaml b/infra/kyverno-policies/kustomization.yaml new file mode 100644 index 0000000..7a7e10d --- /dev/null +++ b/infra/kyverno-policies/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./kube-vip-network-adapter.yaml diff --git a/infra/kyverno/values.yaml b/infra/kyverno/values.yaml index ebd93c3..1150258 100644 --- a/infra/kyverno/values.yaml +++ b/infra/kyverno/values.yaml @@ -1,8 +1,27 @@ admissionController: replicas: 2 + rbac: + clusterRole: + extraResources: + - apiGroups: + - "" + resources: + - "nodes" + verbs: + - get backgroundController: replicas: 2 cleanupController: replicas: 2 reportsController: replicas: 2 + +config: + webhooks: + namespaceSelector: + matchExpressions: [] + + resourceFiltersExclude: + - "[Binding,*,*]" + - "[Pod/binding,*,*]" + - "[*/*,kube-system,*]"