Use kyverno to add annotation to kube-vip pods

clusters/titan.lan.huizinga.dev/infra/kube-vip.yaml

@@ -6,6 +6,8 @@ metadata:
 spec:
   interval: 15m
   path: ./infra/kube-vip
+  dependsOn:
+    - name: kyverno-policies
   prune: true
   timeout: 2m
   sourceRef:

clusters/titan.lan.huizinga.dev/infra/kyverno-policies.yaml

@@ -0,0 +1,16 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: kyverno-policies
+  namespace: flux-system
+spec:
+  interval: 15m
+  path: ./infra/kyverno-policies
+  dependsOn:
+    - name: kyverno
+  prune: true
+  timeout: 2m
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: true

infra/kube-vip/daemon-set-enp3s0.yaml

@@ -1,88 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  labels:
-    app.kubernetes.io/name: kube-vip-ds
-    app.kubernetes.io/version: v0.8.3
-  name: kube-vip-ds-enp3s0
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: kube-vip-ds
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: kube-vip-ds
-        app.kubernetes.io/version: v0.8.3
-    spec:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/master
-                    operator: Exists
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
-      nodeSelector:
-        feature.node.kubernetes.io/network-adapter: enp3s0
-      containers:
-        - args:
-            - manager
-          env:
-            - name: vip_arp
-              value: "true"
-            - name: port
-              value: "6443"
-            - name: vip_nodename
-              valueFrom:
-                fieldRef:
-                  fieldPath: spec.nodeName
-            - name: vip_interface
-              value: enp3s0
-            - name: vip_cidr
-              value: "32"
-            - name: dns_mode
-              value: first
-            - name: cp_enable
-              value: "true"
-            - name: cp_namespace
-              value: kube-system
-            - name: svc_enable
-              value: "true"
-            - name: svc_election
-              value: "true"
-            - name: svc_leasename
-              value: plndr-svcs-lock
-            - name: vip_leaderelection
-              value: "true"
-            - name: vip_leasename
-              value: plndr-cp-lock
-            - name: vip_leaseduration
-              value: "5"
-            - name: vip_renewdeadline
-              value: "3"
-            - name: vip_retryperiod
-              value: "1"
-            - name: address
-              value: 10.0.2.1
-            - name: prometheus_server
-              value: :2112
-          image: ghcr.io/kube-vip/kube-vip:v0.8.3
-          imagePullPolicy: IfNotPresent
-          name: kube-vip
-          resources: {}
-          securityContext:
-            capabilities:
-              add:
-                - NET_ADMIN
-                - NET_RAW
-      hostNetwork: true
-      serviceAccountName: kube-vip
-      tolerations:
-        - effect: NoSchedule
-          operator: Exists
-        - effect: NoExecute
-          operator: Exists
-  updateStrategy: {}

infra/kube-vip/daemon-set.yaml

@@ -2,17 +2,17 @@ apiVersion: apps/v1
 kind: DaemonSet
 metadata:
   labels:
-    app.kubernetes.io/name: kube-vip-ds
+    app.kubernetes.io/name: kube-vip
     app.kubernetes.io/version: v0.8.3
-  name: kube-vip-ds-enp2s0
+  name: kube-vip
 spec:
   selector:
     matchLabels:
-      app.kubernetes.io/name: kube-vip-ds
+      app.kubernetes.io/name: kube-vip
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: kube-vip-ds
+        app.kubernetes.io/name: kube-vip
         app.kubernetes.io/version: v0.8.3
     spec:
       affinity:
@@ -25,8 +25,9 @@ spec:
               - matchExpressions:
                   - key: node-role.kubernetes.io/control-plane
                     operator: Exists
-      nodeSelector:
+              - matchExpressions:
-        feature.node.kubernetes.io/network-adapter: enp2s0
+                  - key: feature.node.kubernetes.io/network-adapter
+                    operator: Exists
       containers:
         - args:
             - manager
@@ -40,7 +41,9 @@ spec:
                 fieldRef:
                   fieldPath: spec.nodeName
             - name: vip_interface
-              value: enp2s0
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.annotations['feature.node.kubernetes.io/network-adapter']
             - name: vip_cidr
               value: "32"
             - name: dns_mode

infra/kube-vip/kustomization.yaml

@@ -5,8 +5,7 @@ resources:
   - ./service-account.yaml
   - ./cluster-role.yaml
   - ./cluster-role-binding.yaml
-  - ./daemon-set-enp2s0.yaml
+  - ./daemon-set.yaml
-  - ./daemon-set-enp3s0.yaml
 
   - https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/refs/tags/v0.0.11/manifest/kube-vip-cloud-controller.yaml
   - ./config-map-kubevip.yaml

infra/kyverno-policies/kube-vip-network-adapter.yaml

@@ -0,0 +1,37 @@
+apiVersion: kyverno.io/v2beta1
+kind: ClusterPolicy
+metadata:
+  name: kube-vip-network-adapter
+  annotations:
+    pod-policies.kyverno.io/autogen-controllers: none
+    policies.kyverno.io/title: Kube VIP adapter label
+    policies.kyverno.io/category: Other
+    policies.kyverno.io/subject: Pod
+    kyverno.io/kyverno-version: 1.10.0
+    policies.kyverno.io/minversion: 1.10.0
+    kyverno.io/kubernetes-version: "1.26"
+spec:
+  background: false
+  rules:
+    - name: add-network-adapter-annotation
+      match:
+        any:
+          - resources:
+              kinds:
+                - Pod/binding
+              names:
+                - kube-vip-*
+      context:
+        - name: node
+          variable:
+            jmesPath: request.object.target.name
+            default: ""
+        - name: adapter
+          apiCall:
+            urlPath: "/api/v1/nodes/{{node}}"
+            jmesPath: 'metadata.labels."feature.node.kubernetes.io/network-adapter" || "empty"'
+      mutate:
+        patchStrategicMerge:
+          metadata:
+            annotations:
+              feature.node.kubernetes.io/network-adapter: "{{ adapter }}"

infra/kyverno-policies/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./kube-vip-network-adapter.yaml

infra/kyverno/values.yaml

@@ -1,8 +1,27 @@
 admissionController:
   replicas: 2
+  rbac:
+    clusterRole:
+      extraResources:
+        - apiGroups:
+            - ""
+          resources:
+            - "nodes"
+          verbs:
+            - get
 backgroundController:
   replicas: 2
 cleanupController:
   replicas: 2
 reportsController:
   replicas: 2
+
+config:
+  webhooks:
+    namespaceSelector:
+      matchExpressions: []
+
+  resourceFiltersExclude:
+    - "[Binding,*,*]"
+    - "[Pod/binding,*,*]"
+    - "[*/*,kube-system,*]"