feat: Added telegram alerts

.gitattributes

@@ -0,0 +1 @@
+deploy.key filter=git-crypt diff=git-crypt

.pre-commit-config.yaml

@@ -0,0 +1,28 @@
+default_install_hook_types: [pre-commit, commit-msg]
+exclude: gotk-.*.yaml
+repos:
+  - repo: builtin
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+        args:
+          - --allow-multiple-documents
+      - id: check-added-large-files
+      - id: check-merge-conflict
+      - id: check-executables-have-shebangs
+
+  - repo: https://github.com/crate-ci/typos
+    rev: v1.40.0
+    hooks:
+      - id: typos
+
+  - repo: https://github.com/sirwart/ripsecrets
+    rev: v0.1.11
+    hooks:
+      - id: ripsecrets-system
+
+  - repo: https://github.com/crate-ci/committed
+    rev: v1.1.8
+    hooks:
+      - id: committed

.secretsignore

@@ -0,0 +1 @@
+deploy.key

.typos.toml

@@ -0,0 +1,6 @@
+[default]
+extend-ignore-re = [
+  "(?Rm)^.*(#|//)\\s*spellchecker:disable-line$",
+  "(?s)(#|//)\\s*spellchecker:off.*?\\n\\s*(#|//)\\s*spellchecker:on",
+  "(#|//)\\s*spellchecker:ignore-next-line\\n.*",
+]

alerts/telegram/alert.yaml

@@ -0,0 +1,18 @@
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Alert
+metadata:
+  name: telegram
+  namespace: flux-system
+spec:
+  providerRef:
+    name: telegram
+  eventSeverity: error
+  eventSources:
+    - kind: GitRepository
+      name: "*"
+    - kind: Kustomization
+      name: "*"
+    - kind: HelmRepository
+      name: "*"
+    - kind: HelmRelease
+      name: "*"

alerts/telegram/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - secret-telegram.yaml
+  - provider.yaml
+  - alert.yaml

alerts/telegram/provider.yaml

@@ -0,0 +1,12 @@
+apiVersion: notification.toolkit.fluxcd.io/v1beta3
+kind: Provider
+metadata:
+  name: telegram
+  namespace: flux-system
+spec:
+  type: telegram
+  address: https://api.telegram.org
+  # TODO: Would be nice if this was not hard coded
+  channel: "-4748034121"
+  secretRef:
+    name: telegram

alerts/telegram/secret-telegram.yaml

@@ -0,0 +1,52 @@
+apiVersion: v1
+stringData:
+    token: ENC[AES256_GCM,data:azBE1dVSbYNZzpagsQrmEoYGIGFn/URQ3VrksQRhUKuJyCrc6o6360u/k38NKg==,iv:iYpqz6Ndh5QESa8yGPU1nKD8/sb3dgloNLa3HFDihnU=,tag:2w0eslkZQOIb+qPYgNmHIQ==,type:str]
+kind: Secret
+metadata:
+    name: telegram
+    namespace: flux-system
+sops:
+    age:
+        - recipient: age1860txadrlqrjwnqh0g466re2nt8jk7xhj640pq9gpsddpg23uynqsp2hul
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBsN2VGQjBoZUp3TjU4YW1W
+            cnJ5QW9aY1FtdUFvcENZRnV5cXVjMkhEOFFjCkFMTm4yUjFZOG56YUVXRkE3SndI
+            eFQ2clF1ZDZicUtCNk9JYW15US9XSDQKLS0tIExHVDdsaVdyVktobWRaUmZPK2V1
+            NSsyVFRaeFJ6NUhOaDdPZy9yOVhVUXMK32YSPm1nDMvCFLbTTVcy3HdSVueUP2lS
+            dNr/myyFX3s8dkXHXJDj14XBu5O3Fk0ktHa2bFfvx+wosHPixG4TqQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1hktythzvsnth6u5en2lvag0tftnj9r03w7rpnzfgzgf5w95qxycq2azufj
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0cDlERWorN1hXNlMxUnVp
+            TjZGcis4eVRBWWZVUG5nRzEwMy9Jd21DSFFvCk1ld2tEZWJvV3pHMmVsb2lDRk53
+            aitnbmJpaFE5NGxGd0N3VSt1N2N2WHcKLS0tIDRKMTN0bk9hcUNTNWpsaEMrY01R
+            ckU4YlF3Z0JXTGRaVnJnd1ZITHU3RzQKk/kNn84JwoWl/77wEyeljrTkYSTBSpAW
+            fDn4ddsrmays2kEb8ux0ZbqsNNrt3IhDYd0urks20INvS9BkXJMI0A==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2025-12-09T01:52:28Z"
+    mac: ENC[AES256_GCM,data:EObi2Da0n+RpKOfpay2NQM140lniPUY12MNFQIw4wNlCS8EFzJHNrPk/SpFDOY5HqvT27wRfz//oTGVzt6+TNkfAU3GcGYuJZezwFvbAkkoRuVmGauZ/ZyfG8PmdPN1imYa2ajqaQW/gym06TbtaLaI54txeK0PE1z+oWXZIW14=,iv:fVU6mkfyuck/pu7ff5n2Gd6pGclLU7jt5Dy2vOOnctA=,tag:4JUDNvgVskKlAHlJ5MwsFA==,type:str]
+    pgp:
+        - created_at: "2025-12-09T01:52:28Z"
+          enc: |-
+            -----BEGIN PGP MESSAGE-----
+
+            hQIMA7pKPTYH5bqOARAAlVFcHIEsMcSUuPWKxOtpGBLdnWo/9d8GsQwUDsh9Kk8C
+            tLVJn8cXNGdvt28pmS22L3KDOX0o3WGNNWvZfe9D6cG4WPH1yug4axQoVTnMUpYE
+            a8nsqMRgfNdTkbDfOtU+EV4Aq9iMl2MVqy3D/Vc2MnPk/WvcQUGXd1viJJrOhBcP
+            S0mTDRQOUjdgrX4RIcinx2yQQgMTAiduxj0sSaK6BWEDaum50WjthV+/GlCz2hks
+            nf+BzY+MbCDFKMjfSbpdZhyQ0VvkV8Yu7UtoxaI6w9KUqH8zHjqaNjhELmLy2FKv
+            6kAiW/XUuR6BZTbo1MgLG4gJwQU6+pHELXAh/h+Qj+wX6agYOZNmQqAmQahnfQZ/
+            cy7mzGf4vQlMxeayOUdJVDbqhWlMpCVmGkWqtfBwAJKr7r7fjRP+VuZ06KyilGr0
+            NyibyKVp6+ostTXNlMmvTGBiQpNv+/9822PhFKN/ARde32igUmtnRiwoYyrZ644y
+            Y6BAmnEuu+mfpIDPODQ4gWP+t9vwuzBPTUP2DVqXgqHoOgK2HnGIeNtKh8PqPkG4
+            qPI/3b1LqN+yMOI9HU1fv2IUbRPksauNNq1rrK/9scasT1MqoKEBWjntdDJ7MuR6
+            RQ6S3xU8dyBW0jNe3JFa1dIvM+QlVysIpy8kCv81DeNDT/Yp8psTJXeBa25tcovS
+            XAHlK1OXm7zOchfPa46EWjbMMSwcIle9VVrJCAsjS2eFriOK8wMn3Awyl9V2zcEX
+            6F6+jzvVYUbxXZupCdJoZo7faaC8TyA10o2LxrcdFwLtjByVgVz3mYmpHVwQ
+            =BwkI
+            -----END PGP MESSAGE-----
+          fp: 1E0CF38FF7C9ADAED58B436ABA4A3D3607E5BA8E!
+    encrypted_regex: ^(data|stringData)$
+    version: 3.11.0

bootstrap.sh

@@ -58,6 +58,7 @@ if [ "${vip}" = "null" ]; then
 fi
 
 echo -n "Checking connection to ${bootstrap_ip}... "
+# spellchecker:ignore-next-line
 if nmap -Pn ${bootstrap_ip} -p 50000 | grep -q 'open'; then
 	echo "[Success]"
 else
@@ -67,6 +68,7 @@ fi
 
 count=0
 max_retries=20
+# spellchecker:ignore-next-line
 while ! nmap -Pn ${vip} -p 50000 | grep -q 'open' && [ ${count} -lt ${max_retries} ]; do
 	if [ $count -eq 0 ]; then
 		echo -n "Bootstrapping Kubernetes"
@@ -122,5 +124,9 @@ cilium-cli status --wait
 # cilium-cli connectivity test --namespace-labels pod-security.kubernetes.io/enforce=privileged
 
 echo "Bootstrapping flux..."
-flux bootstrap git --url ssh://git@huizinga.dev/infra/foundation --branch=main --path=clusters/${cluster_name} \
+flux bootstrap git \
+	--url ssh://git@huizinga.dev/infra/foundation \
+	--branch=main \
+	--private-key-file=clusters/${cluster_name}/deploy.key -s \
+	--path=clusters/${cluster_name} \
 	--components-extra=source-watcher

clusters/testing/deploy.key.pub

@@ -0,0 +1 @@
+ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK59NC6hLyDf+9zeOQ0stZeay51UyUpoBgONh0xxJFIlgra5ojyhrrQVlfjcUqdLe5yijWU1nCxKpaFGDPMdNE4= flux@testing

clusters/testing/kustomization.yaml

@@ -2,9 +2,16 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - flux-system/
+
   - ../../controllers/artifacts.yaml
   - ../../controllers/cilium/cilium.yaml
   - ../../controllers/cert-manager/cert-manager.yaml
+  - ../../controllers/spegel/spegel.yaml
+  - ../../controllers/longhorn/longhorn.yaml
+  - ../../controllers/local-path-provisioner/local-path-provisioner.yaml
+
   - ../../configs/artifacts.yaml
   - ../../configs/letsencrypt/letsencrypt.yaml
   - ../../configs/certificates/certificates.yaml
+
+  - ../../alerts/telegram

committed.toml

@@ -0,0 +1,2 @@
+style = "conventional"
+ignore_author_re = "Flux"

controllers/artifacts.yaml

@@ -19,3 +19,23 @@ spec:
       copy:
         - from: "@foundation/controllers/cert-manager/**"
           to: "@artifact/"
+    - name: spegel
+      originRevision: "@foundation"
+      copy:
+        - from: "@foundation/controllers/spegel/**"
+          to: "@artifact/"
+    - name: openebs
+      originRevision: "@foundation"
+      copy:
+        - from: "@foundation/controllers/openebs/**"
+          to: "@artifact/"
+    - name: longhorn
+      originRevision: "@foundation"
+      copy:
+        - from: "@foundation/controllers/longhorn/**"
+          to: "@artifact/"
+    - name: local-path-provisioner
+      originRevision: "@foundation"
+      copy:
+        - from: "@foundation/controllers/local-path-provisioner**"
+          to: "@artifact/"

controllers/local-path-provisioner/kustomization.yaml

@@ -0,0 +1,28 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - github.com/rancher/local-path-provisioner/deploy?ref=v0.0.32
+patches:
+  - patch: |-
+      kind: ConfigMap
+      apiVersion: v1
+      metadata:
+        name: local-path-config
+        namespace: local-path-storage
+      data:
+        config.json: |-
+          {
+                  "nodePathMap":[
+                  {
+                          "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES",
+                          "paths":["/var/mnt/local-path-provisioner"]
+                  }
+                  ]
+          }
+  - patch: |-
+      apiVersion: v1
+      kind: Namespace
+      metadata:
+        name: local-path-storage
+        labels:
+          pod-security.kubernetes.io/enforce: privileged

controllers/local-path-provisioner/local-path-provisioner.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: local-path-provisioner
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: ExternalArtifact
+    name: local-path-provisioner
+  path: ./
+  prune: true
+  wait: true

controllers/longhorn/base/helm-release.yaml

@@ -0,0 +1,30 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 12h
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 2m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 3m
+  chart:
+    spec:
+      chart: longhorn
+      # TODO: Is it safe to upgrade automatically to 1.x?
+      version: "1.10.x"
+      sourceRef:
+        kind: HelmRepository
+        name: longhorn
+      interval: 24h
+  valuesFrom:
+    - kind: ConfigMap
+      name: values-base
+    - kind: ConfigMap
+      name: values-overlay
+      optional: true

controllers/longhorn/base/helm-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: longhorn
+  namespace: longhorn-system
+spec:
+  interval: 24h
+  url: https://charts.longhorn.io

controllers/longhorn/base/kustomization.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repository.yaml
+  - helm-release.yaml
+
+configurations:
+  - name-reference.yaml
+
+configMapGenerator:
+  - name: values-base
+    namespace: cilium
+    files:
+      - values.yaml

controllers/longhorn/base/name-reference.yaml

@@ -0,0 +1,6 @@
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease

controllers/longhorn/base/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: longhorn-system
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
+    pod-security.kubernetes.io/audit: privileged

controllers/longhorn/base/values.yaml

@@ -0,0 +1,10 @@
+defaultSettings:
+  defaultDataPath: /var/mnt/longhorn
+  replicaAutoBalance: best-effort
+  defaultDataLocality: best-effort
+  freezeFilesystemForSnapshot: enabled
+  storageReservedPercentageForDefaultDisk: 0
+  storageMinimalAvailablePercentage: 10
+  storageOverProvisioningPercentage: 25
+persistence:
+  defaultDataLocality: best-effort

controllers/longhorn/longhorn.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: longhorn
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: ExternalArtifact
+    name: longhorn
+  path: ./${cluster_env}
+  prune: true
+  wait: true

controllers/longhorn/production/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+
+configMapGenerator:
+  - name: values-overlay
+    namespace: cilium
+    files:
+      - values.yaml

controllers/longhorn/production/values.yaml

@@ -0,0 +1,4 @@
+defaultSettings:
+  defaultReplicaCount: 2
+persistence:
+  defaultClassReplicaCount: 2

controllers/longhorn/testing/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+
+configMapGenerator:
+  - name: values-overlay
+    namespace: cilium
+    files:
+      - values.yaml

controllers/longhorn/testing/values.yaml

@@ -0,0 +1,4 @@
+defaultSettings:
+  defaultReplicaCount: 1
+persistence:
+  defaultClassReplicaCount: 1

controllers/openebs/helm-release.yaml

@@ -0,0 +1,49 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: openebs
+  namespace: openebs
+spec:
+  interval: 12h
+  install:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 2m
+  upgrade:
+    strategy:
+      name: RetryOnFailure
+      retryInterval: 3m
+  chart:
+    spec:
+      chart: openebs
+      version: "4.x"
+      sourceRef:
+        kind: HelmRepository
+        name: openebs
+      interval: 24h
+  values:
+    loki:
+      enabled: false
+    alloy:
+      enabled: false
+    mayastor:
+      etcd:
+        replicaCount: 1
+      nats:
+        cluster:
+          replicas: 1
+      eventing:
+        enabled: false
+      obs:
+        callhome:
+          enabled: false
+      csi:
+        node:
+          initContainers:
+            enabled: false
+    engines:
+      local:
+        lvm:
+          enabled: false
+        zfs:
+          enabled: false

controllers/openebs/helm-repository.yaml

@@ -0,0 +1,8 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: openebs
+  namespace: openebs
+spec:
+  interval: 24h
+  url: https://openebs.github.io/openebs

controllers/openebs/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repository.yaml
+  - helm-release.yaml

controllers/openebs/namespace.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: openebs
+  labels:
+    pod-security.kubernetes.io/enforce: privileged
+    pod-security.kubernetes.io/warn: privileged
+    pod-security.kubernetes.io/audit: privileged

controllers/openebs/openebs.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: openebs
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: ExternalArtifact
+    name: openebs
+  path: ./
+  prune: true
+  wait: true

controllers/spegel/helm-release.yaml

@@ -0,0 +1,17 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: spegel
+  namespace: spegel
+spec:
+  interval: 1m
+  chart:
+    spec:
+      chart: spegel
+      interval: 5m
+      sourceRef:
+        kind: HelmRepository
+        name: spegel
+  values:
+    spegel:
+      containerdRegistryConfigPath: /etc/cri/conf.d/hosts

controllers/spegel/helm-repository.yaml

@@ -0,0 +1,9 @@
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: spegel
+  namespace: spegel
+spec:
+  type: "oci"
+  interval: 5m0s
+  url: oci://ghcr.io/spegel-org/helm-charts

controllers/spegel/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-repository.yaml
+  - helm-release.yaml

controllers/spegel/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: spegel
+  labels:
+    pod-security.kubernetes.io/enforce: privileged

controllers/spegel/spegel.yaml

@@ -0,0 +1,15 @@
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: spegel
+  namespace: flux-system
+spec:
+  interval: 1h
+  retryInterval: 2m
+  timeout: 5m
+  sourceRef:
+    kind: ExternalArtifact
+    name: spegel
+  path: ./
+  prune: true
+  wait: true