feat(clustertool): BREAKING CHANGE move to nginx

clustertool/embed/generic/base/clusterenv.yaml

@@ -8,8 +8,10 @@ MASTER1IP: 192.168.20.210
 GATEWAY: 192.168.20.1
 # Defines the ip range metallb is allowed to use
 METALLB_RANGE: 192.168.20.211-192.168.20.219
-# IP Traefik will use for externally exposing ingress-resources (NOT its dashboard)
+# IP NGINX will use for exposing ingress-resources. Internal refers to LAN-only NGINX ingress, dont port-forward at router
-TRAEFIK_IP: 192.168.20.211
+NGINX_INTERNAL_IP: 192.168.20.221
+# IP NGINX will use for exposing ingress-resources. Internal refers to LAN-and-WAN NGINX ingress, DO port-forward at router
+NGINX_EXTERNAL_IP: 192.168.20.222
 # IP Blocky DNS will be exposed on
 BLOCKY_IP: 192.168.20.212
 # Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use

clustertool/embed/generic/kubernetes/core/system-upgrade-controller-plans/app/talos.yaml

@@ -30,7 +30,7 @@ spec:
       operator: Exists
       effect: NoSchedule
   upgrade:
-    image: ghcr.io/jfroy/tnu:0.3.0
+    image: ghcr.io/jfroy/tnu:0.4.1
     envs:
       - name: NODE
         valueFrom:

clustertool/embed/generic/kubernetes/core/traefik/app/helm-release.yaml

@@ -1,40 +0,0 @@
----
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: traefik
-  namespace: traefik
-spec:
-  interval: 15m
-  chart:
-    spec:
-      chart: traefik
-      version: 30.4.3
-      sourceRef:
-        kind: HelmRepository
-        name: truecharts
-        namespace: flux-system
-      interval: 15m
-  timeout: 20m
-  maxHistory: 3
-  driftDetection:
-    mode: warn
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values:
-    ## set stopAll to "true" if you do not want to use traefik
-    # and/or disable its service
-    global:
-      stopAll: false
-    defaultCertificate: "domain-0-wildcard"
-    service:
-      tcp:
-        loadBalancerIP: ${TRAEFIK_IP}

clustertool/embed/generic/kubernetes/core/traefik/app/namespace.yaml

@@ -1,6 +0,0 @@
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: traefik
-  labels:
-    topolvm.io/webhook: ignore

clustertool/embed/generic/kubernetes/kube-system/descheduler/app/bootstrap-values.yaml.ct

@@ -0,0 +1,3 @@
+## DO NOT ALTER THIS FILE, CHANGE DO NOT PERSIST. Alter TalEnv.yaml instead.
+serviceMonitor:
+  enabled: false

clustertool/embed/generic/kubernetes/kube-system/descheduler/app/helm-release.yaml

@@ -1,67 +1,76 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: descheduler
+    name: descheduler
-  namespace: kube-system
+    namespace: kube-system
 spec:
-  interval: 15m
+    interval: 15m
-  chart:
+    chart:
-    spec:
+        spec:
-      chart: descheduler
+            chart: descheduler
-      version: 0.1.0
+            version: 0.32.2
-      sourceRef:
+            sourceRef:
-        kind: HelmRepository
+                kind: HelmRepository
-        name: truecharts
+                name: home-ops-mirror
-        namespace: flux-system
+                namespace: flux-system
-      interval: 15m
+            interval: 15m
-  timeout: 20m
+    timeout: 20m
-  maxHistory: 3
+    maxHistory: 3
-  driftDetection:
+    install:
-    mode: warn
+        createNamespace: true
-  install:
+        remediation:
-    createNamespace: true
+            retries: 3
-    remediation:
+    upgrade:
-      retries: 3
+        cleanupOnFail: true
-  upgrade:
+        remediation:
-    cleanupOnFail: true
+            retries: 3
-    remediation:
+    uninstall:
-      retries: 3
+        keepHistory: false
-  uninstall:
+    values:
-    keepHistory: false
+      replicas: 2
-  values:
+      kind: Deployment
-    kind: Deployment
+      deschedulerPolicyAPIVersion: descheduler/v1alpha2
-    deschedulerPolicy:
+      deschedulerPolicy:
-      strategies:
+        profiles:
-        RemoveDuplicates:
+          - name: Default
-          enabled: true
+            pluginConfig:
-        RemovePodsViolatingNodeTaints:
+              - name: DefaultEvictor
-          enabled: true
+                args:
-        RemovePodsViolatingNodeAffinity:
+                  evictFailedBarePods: true
-          enabled: true
+                  evictLocalStoragePods: true
-          params:
+                  evictSystemCriticalPods: true
-            nodeAffinityType:
+              - name: RemoveFailedPods
-              - requiredDuringSchedulingIgnoredDuringExecution
+                args:
-        RemovePodsViolatingTopologySpreadConstraint:
+                  reasons:
-          enabled: true
+                    - ContainerStatusUnknown
-          params:
+                    - NodeAffinity
-            includeSoftConstraints: true
+                    - NodeShutdown
-        RemovePodsViolatingInterPodAntiAffinity:
+                    - Terminated
-          enabled: true
+                    - UnexpectedAdmissionError
-          params:
+                  includingInitContainers: true
-            nodeFit: true
+                  excludeOwnerKinds:
-        LowNodeUtilization:
+                    - Job
-          enabled: false
+                  minPodLifetimeSeconds: 1800
-        RemoveFailedPods:
+              - name: RemovePodsViolatingInterPodAntiAffinity
-          enabled: true
+              - name: RemovePodsViolatingNodeAffinity
-          params:
+                args:
-            failedPods:
+                  nodeAffinityType:
-              includingInitContainers: true
+                    - requiredDuringSchedulingIgnoredDuringExecution
-              excludeOwnerKinds:
+              - name: RemovePodsViolatingNodeTaints
-                - Job
+              - name: RemovePodsViolatingTopologySpreadConstraint
-              minPodLifetimeSeconds: 3600
+            plugins:
-        RemovePodsHavingTooManyRestarts:
+              balance:
-          enabled: true
+                enabled:
-          params:
+                  - RemovePodsViolatingTopologySpreadConstraint
-            podsHavingTooManyRestarts:
+              deschedule:
-              podRestartThreshold: 100
+                enabled:
-              includingInitContainers: true
+                  - RemoveFailedPods
+                  - RemovePodsViolatingInterPodAntiAffinity
+                  - RemovePodsViolatingNodeAffinity
+                  - RemovePodsViolatingNodeTaints
+      service:
+        enabled: true
+      serviceMonitor:
+        enabled: false
+      leaderElection:
+        enabled: true

clustertool/embed/generic/kubernetes/networking/nginx-external/app/helm-release.yaml

@@ -0,0 +1,85 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: nginx-external
+  namespace: nginx-external
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: 'ingress-nginx'
+      version: 4.12.1
+      sourceRef:
+        kind: HelmRepository
+        name: 'ingress-nginx'
+        namespace: flux-system
+      interval: 15m
+  timeout: 20m
+  maxHistory: 3
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controller:
+      replicaCount: 2
+      service:
+        externalTrafficPolicy: Local
+        annotations:
+          # TODO: Maybe play with this
+          # internal-dns.alpha.kubernetes.io/hostname: external.${DOMAIN_0}
+          metallb.io/ip-allocated-from-pool: main
+          metallb.io/loadBalancerIPs: ${NGINX_EXTERNAL_IP}
+          metallb.universe.tf/ip-allocated-from-pool: main
+          "lbipam.cilium.io/ips": ${NGINX_EXTERNAL_IP}
+      ingressClassByName: true
+      watchIngressWithoutClass: false
+      ingressClassResource:
+        name: external
+        default: false
+        controllerValue: k8s.io/external
+      config:
+        allow-snippet-annotations: true
+        annotations-risk-level: Critical
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-ocsp: "true"
+        enable-real-ip: "true"
+        force-ssl-redirect: "true"
+        hide-headers: Server,X-Powered-By
+        hsts-max-age: "31449600"
+        keep-alive-requests: 10000
+        keep-alive: 120
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+        use-forwarded-headers: "true"
+      metrics:
+        enabled: true
+        # serviceMonitor:
+        #   enabled: true
+        #   namespaceSelector:
+        #     any: true
+      extraArgs:
+        default-ssl-certificate: "clusterissuer/certificate-issuer-domain-0-wildcard"
+        publish-status-address: ${NGINX_EXTERNAL_IP}
+      terminationGracePeriodSeconds: 120
+      publishService:
+        enabled: false
+      resources:
+        requests:
+          cpu: 100m
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false

clustertool/embed/generic/kubernetes/networking/nginx-external/app/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+    name: nginx-external

clustertool/embed/generic/kubernetes/networking/nginx-internal/app/helm-release.yaml

@@ -0,0 +1,85 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: nginx-internal
+  namespace: nginx-internal
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: 'ingress-nginx'
+      version: 4.12.1
+      sourceRef:
+        kind: HelmRepository
+        name: 'ingress-nginx'
+        namespace: flux-system
+      interval: 15m
+  timeout: 20m
+  maxHistory: 3
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    controller:
+      replicaCount: 2
+      service:
+        externalTrafficPolicy: Local
+        annotations:
+          # TODO: Maybe play with this
+          # internal-dns.alpha.kubernetes.io/hostname: internal.${DOMAIN_0}
+          metallb.io/ip-allocated-from-pool: main
+          metallb.io/loadBalancerIPs: ${NGINX_INTERNAL_IP}
+          metallb.universe.tf/ip-allocated-from-pool: main
+          "lbipam.cilium.io/ips": ${NGINX_INTERNAL_IP}
+      ingressClassByName: true
+      watchIngressWithoutClass: true
+      ingressClassResource:
+        name: internal
+        default: true
+        controllerValue: k8s.io/internal
+      config:
+        allow-snippet-annotations: true
+        annotations-risk-level: Critical
+        client-body-buffer-size: 100M
+        client-body-timeout: 120
+        client-header-timeout: 120
+        enable-brotli: "true"
+        enable-ocsp: "true"
+        enable-real-ip: "true"
+        force-ssl-redirect: "true"
+        hide-headers: Server,X-Powered-By
+        hsts-max-age: "31449600"
+        keep-alive-requests: 10000
+        keep-alive: 120
+        proxy-body-size: 0
+        proxy-buffer-size: 16k
+        ssl-protocols: TLSv1.3 TLSv1.2
+        use-forwarded-headers: "true"
+      metrics:
+        enabled: true
+        # serviceMonitor:
+        #   enabled: true
+        #   namespaceSelector:
+        #     any: true
+      extraArgs:
+        default-ssl-certificate: "clusterissuer/certificate-issuer-domain-0-wildcard"
+        publish-status-address: ${NGINX_INTERNAL_IP}
+      terminationGracePeriodSeconds: 120
+      publishService:
+        enabled: false
+      resources:
+        requests:
+          cpu: 100m
+        limits:
+          memory: 500Mi
+    defaultBackend:
+      enabled: false

clustertool/embed/generic/kubernetes/networking/nginx-internal/app/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+    name: nginx-internal

clustertool/embed/generic/kubernetes/system/cert-manager/app/helm-release.yaml

@@ -1,26 +1,30 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: cert-manager
+    name: cert-manager
-  namespace: cert-manager
+    namespace: cert-manager
 spec:
-  interval: 5m
+    interval: 5m
-  chart:
+    chart:
-    spec:
+        spec:
-      chart: cert-manager
+            chart: cert-manager
-      version: 6.5.3
+            version: v1.17.1
-      sourceRef:
+            sourceRef:
-        kind: HelmRepository
+                kind: HelmRepository
-        name: truecharts
+                name: home-ops-mirror
-        namespace: flux-system
+                namespace: flux-system
-      interval: 5m
+            interval: 5m
-  install:
+    install:
-    createNamespace: true
+        createNamespace: true
-    crds: CreateReplace
+        crds: CreateReplace
-    remediation:
+        remediation:
-      retries: 3
+            retries: 3
-  upgrade:
+    upgrade:
-    crds: CreateReplace
+        crds: CreateReplace
-    remediation:
+        remediation:
-      retries: 3
+            retries: 3
-  values:
+    values:
+      dns01RecursiveNameservers: "1.1.1.1:53,1.0.0.1:53"
+      dns01RecursiveNameserversOnly: false
+      installCRDs: true
+      enableCertificateOwnerRef: true

clustertool/embed/generic/kubernetes/system/cloudnative-pg/app/helm-release.yaml

@@ -1,26 +1,33 @@
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
-  name: cloudnative-pg
+    name: cloudnative-pg
-  namespace: cloudnative-pg
+    namespace: cloudnative-pg
 spec:
-  interval: 5m
+    interval: 5m
-  chart:
+    chart:
-    spec:
+        spec:
-      chart: cloudnative-pg
+            chart: cloudnative-pg
-      version: 8.2.0
+            version: 0.23.2
-      sourceRef:
+            sourceRef:
-        kind: HelmRepository
+                kind: HelmRepository
-        name: truecharts
+                name: cloudnative-pg
-        namespace: flux-system
+                namespace: flux-system
-      interval: 5m
+            interval: 5m
-  install:
+    install:
-    createNamespace: true
+        createNamespace: true
-    crds: CreateReplace
+        crds: CreateReplace
-    remediation:
+        remediation:
-      retries: 3
+            retries: 3
-  upgrade:
+    upgrade:
-    crds: CreateReplace
+        crds: CreateReplace
-    remediation:
+        remediation:
-      retries: 3
+            retries: 3
-  values:
+    values:
+      crds:
+        create: true
+      replicaCount: 2
+      # monitoring:
+      #   podMonitorEnabled: false
+      #   grafanaDashboard:
+      #     create: true

clustertool/embed/generic/kubernetes/system/kube-prometheus-stack/app/alertmanagerconfig.yaml

@@ -0,0 +1,86 @@
+# ---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/alertmanagerconfig_v1alpha1.json
+# apiVersion: monitoring.coreos.com/v1alpha1
+# kind: AlertmanagerConfig
+# metadata:
+#   name: alertmanager
+# spec:
+#   route:
+#     groupBy: ["alertname", "job"]
+#     groupInterval: 10m
+#     groupWait: 1m
+#     receiver: pushover
+#     repeatInterval: 12h
+#     routes:
+#       - receiver: "null"
+#         matchers:
+#           - name: alertname
+#             value: InfoInhibitor
+#             matchType: =
+#       - receiver: heartbeat
+#         groupInterval: 5m
+#         groupWait: 0s
+#         repeatInterval: 5m
+#         matchers:
+#           - name: alertname
+#             value: Watchdog
+#             matchType: =
+#       - receiver: pushover
+#         matchers:
+#           - name: severity
+#             value: critical
+#             matchType: =
+#   inhibitRules:
+#     - equal: ["alertname", "namespace"]
+#       sourceMatch:
+#         - name: severity
+#           value: critical
+#           matchType: =
+#       targetMatch:
+#         - name: severity
+#           value: warning
+#           matchType: =
+#   receivers:
+#     - name: "null"
+#     - name: heartbeat
+#       webhookConfigs:
+#         - urlSecret:
+#             name: &secret alertmanager-secret
+#             key: ALERTMANAGER_HEARTBEAT_URL
+#     - name: pushover
+#       pushoverConfigs:
+#         - html: true
+#           message: |-
+#             {{- range .Alerts }}
+#               {{- if ne .Annotations.description "" }}
+#                 {{ .Annotations.description }}
+#               {{- else if ne .Annotations.summary "" }}
+#                 {{ .Annotations.summary }}
+#               {{- else if ne .Annotations.message "" }}
+#                 {{ .Annotations.message }}
+#               {{- else }}
+#                 Alert description not available
+#               {{- end }}
+#               {{- if gt (len .Labels.SortedPairs) 0 }}
+#                 <small>
+#                   {{- range .Labels.SortedPairs }}
+#                     <b>{{ .Name }}:</b> {{ .Value }}
+#                   {{- end }}
+#                 </small>
+#               {{- end }}
+#             {{- end }}
+#           priority: |-
+#             {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+#           sendResolved: true
+#           sound: gamelan
+#           title: >-
+#             [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
+#             {{ .CommonLabels.alertname }}
+#           ttl: 86400s
+#           token:
+#             name: *secret
+#             key: ALERTMANAGER_PUSHOVER_TOKEN
+#           userKey:
+#             name: *secret
+#             key: PUSHOVER_USER_KEY
+#           urlTitle: View in Alertmanager

clustertool/embed/generic/kubernetes/system/kube-prometheus-stack/app/helm-release.yaml

@@ -0,0 +1,112 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: kube-prometheus-stack
+  namespace: kube-prometheus-stack
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: kube-prometheus-stack
+      version: 69.8.2
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+      interval: 15m
+  timeout: 20m
+  maxHistory: 3
+  install:
+    createNamespace: true
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  values:
+    crds:
+      enabled: true
+      upgradeJob:
+        enabled: true
+        forceConflicts: true
+    cleanPrometheusOperatorObjectNames: true
+    alertmanager:
+      enabled: false
+    kubeEtcd:
+      service:
+        selector:
+          component: kube-apiserver
+    kubeProxy:
+      enabled: false
+    prometheus:
+      ingress:
+        enabled: false
+      prometheusSpec:
+        disableCompaction: true
+        podMonitorSelectorNilUsesHelmValues: false
+        probeSelectorNilUsesHelmValues: false
+        ruleSelectorNilUsesHelmValues: false
+        scrapeConfigSelectorNilUsesHelmValues: false
+        serviceMonitorSelectorNilUsesHelmValues: false
+        enableAdminAPI: true
+        walCompression: true
+        enableFeatures:
+          - memory-snapshot-on-shutdown
+        # Prometheus metrics retention time in PV
+        retention: 24h
+        retentionSize: 50GB
+        # External labels to add to any time series or
+        # alerts when communicating with external systems
+        externalLabels:
+          cluster: cluster_main
+          env: prod
+        # High availability
+        replicas: 2
+        resources:
+          requests:
+            cpu: 100m
+          limits:
+            memory: 2000Mi
+        # storageSpec:
+        #   volumeClaimTemplate:
+        #     spec:
+        #       resources:
+        #         requests:
+        #           storage: 60Gi
+
+    prometheus-node-exporter:
+      fullnameOverride: node-exporter
+      prometheus:
+        monitor:
+          enabled: true
+          relabelings:
+            - action: replace
+              regex: (.*)
+              replacement: $1
+              sourceLabels: ["__meta_kubernetes_pod_node_name"]
+              targetLabel: kubernetes_node
+    kube-state-metrics:
+      fullnameOverride: kube-state-metrics
+      # metricLabelsAllowlist:
+      #   - pods=[*]
+      #   - deployments=[*]
+      #   - persistentvolumeclaims=[*]
+      prometheus:
+        monitor:
+          enabled: true
+          relabelings:
+            - action: replace
+              regex: (.*)
+              replacement: $1
+              sourceLabels: ["__meta_kubernetes_pod_node_name"]
+              targetLabel: kubernetes_node
+    grafana:
+      enabled: false
+      forceDeployDashboards: true
+      defaultDashboardsEnabled: true
+      forceDeployDatasources: true

clustertool/embed/generic/kubernetes/system/kube-prometheus-stack/app/kustomization.yaml

@@ -0,0 +1,7 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - helm-release.yaml
+  - alertmanagerconfig.yaml

clustertool/embed/generic/kubernetes/system/kube-prometheus-stack/app/namespace.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+    name: kube-prometheus-stack
+    labels:
+        pod-security.kubernetes.io/enforce: privileged

clustertool/embed/generic/kubernetes/system/prometheus-operator/app/helm-release.yaml

@@ -1,26 +0,0 @@
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: prometheus-operator
-  namespace: prometheus-operator
-spec:
-  interval: 5m
-  chart:
-    spec:
-      chart: prometheus-operator
-      version: 11.5.1
-      sourceRef:
-        kind: HelmRepository
-        name: truecharts
-        namespace: flux-system
-      interval: 5m
-  install:
-    createNamespace: true
-    crds: CreateReplace
-    remediation:
-      retries: 3
-  upgrade:
-    crds: CreateReplace
-    remediation:
-      retries: 3
-  values: {}

clustertool/embed/generic/kubernetes/system/system-upgrade-controller/app/helm-release.yaml

@@ -10,7 +10,7 @@ spec:
   chart:
     spec:
       chart: app-template
-      version: 3.7.3
+      version: 3.5.1
       sourceRef:
         kind: HelmRepository
         name: bjw-s
@@ -31,14 +31,14 @@ spec:
           app:
             image:
               repository: docker.io/rancher/system-upgrade-controller
-              tag: v0.15.2@sha256:3e899833afcea9a8788d384ce976df9a05be84636fe5c01ec2307b5bd8fe9810
+              tag: v0.14.1@sha256:7e13a9b2b984f0c0fd6328439b575348723cc6954b91db3453057fcb784e2d29
             env:
               SYSTEM_UPGRADE_CONTROLLER_DEBUG: false
               SYSTEM_UPGRADE_CONTROLLER_THREADS: 2
               SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900
               SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99
               SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent
-              SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: registry.k8s.io/kubectl:v1.32.3
+              SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: registry.k8s.io/kubectl:v1.31.1
               SYSTEM_UPGRADE_JOB_POD_REPLACEMENT_POLICY: Failed
               SYSTEM_UPGRADE_JOB_PRIVILEGED: true
               SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900

clustertool/embed/generic/kubernetes/system/system-upgrade-controller/app/namespace.yaml

@@ -1,10 +1,10 @@
 apiVersion: v1
 kind: Namespace
 metadata:
-  name: system-upgrade
+    name: system-upgrade
-  annotations:
+    annotations:
-    volsync.backube/privileged-movers: "true"
+        volsync.backube/privileged-movers: "true"
-  labels:
+    labels:
-    kustomize.toolkit.fluxcd.io/prune: disabled
+        kustomize.toolkit.fluxcd.io/prune: disabled
-    goldilocks.fairwinds.com/enabled: "true"
+        goldilocks.fairwinds.com/enabled: "true"
-    pod-security.kubernetes.io/enforce: privileged
+        pod-security.kubernetes.io/enforce: privileged

clustertool/embed/generic/kubernetes/system/traefik-crds/app/helm-release.yaml

@@ -1,32 +0,0 @@
-# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: traefik-crds
-  namespace: system
-spec:
-  interval: 15m
-  chart:
-    spec:
-      chart: traefik-crds
-      version: 3.5.1
-      sourceRef:
-        kind: HelmRepository
-        name: truecharts
-        namespace: flux-system
-      interval: 15m
-  timeout: 20m
-  maxHistory: 3
-  driftDetection:
-    mode: warn
-  install:
-    createNamespace: true
-    remediation:
-      retries: 3
-  upgrade:
-    cleanupOnFail: true
-    remediation:
-      retries: 3
-  uninstall:
-    keepHistory: false
-  values: {}

clustertool/embed/generic/root/repositories/helm/home-ops-mirror.yaml

@@ -0,0 +1,11 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1.json
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: home-ops-mirror
+  namespace: flux-system
+spec:
+  type: oci
+  interval: 2h
+  url: oci://ghcr.io/home-operations/charts-mirror/

clustertool/embed/generic/root/repositories/helm/kustomization.yaml

@@ -10,6 +10,7 @@ resources:
   - coredns.yaml
   - csi-driver-nfs.yaml
   - descheduler.yaml
+  - home-ops-mirror.yaml
   - intel.yaml
   - jetstack.yaml
   - kubernetes-sigs-metrics-server.yaml