feat(clustertool): BREAKING CHANGE move to nginx

This commit is contained in:
Kjeld Schouten
2025-04-03 17:37:59 +02:00
parent ad5503debe
commit 5e9554a3ce
22 changed files with 546 additions and 224 deletions
@@ -8,8 +8,10 @@ MASTER1IP: 192.168.20.210
GATEWAY: 192.168.20.1
# Defines the ip range metallb is allowed to use
METALLB_RANGE: 192.168.20.211-192.168.20.219
# IP Traefik will use for externally exposing ingress-resources (NOT its dashboard)
TRAEFIK_IP: 192.168.20.211
# IP NGINX will use for exposing ingress-resources. Internal refers to LAN-only NGINX ingress, dont port-forward at router
NGINX_INTERNAL_IP: 192.168.20.221
# IP NGINX will use for exposing ingress-resources. Internal refers to LAN-and-WAN NGINX ingress, DO port-forward at router
NGINX_EXTERNAL_IP: 192.168.20.222
# IP Blocky DNS will be exposed on
BLOCKY_IP: 192.168.20.212
# Sets the Kubernetes Dashboard IP. Has to be within METALLB_RANGE and not in use
@@ -30,7 +30,7 @@ spec:
operator: Exists
effect: NoSchedule
upgrade:
image: ghcr.io/jfroy/tnu:0.3.0
image: ghcr.io/jfroy/tnu:0.4.1
envs:
- name: NODE
valueFrom:
@@ -1,40 +0,0 @@
---
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: traefik
namespace: traefik
spec:
interval: 15m
chart:
spec:
chart: traefik
version: 30.4.3
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
driftDetection:
mode: warn
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
## set stopAll to "true" if you do not want to use traefik
# and/or disable its service
global:
stopAll: false
defaultCertificate: "domain-0-wildcard"
service:
tcp:
loadBalancerIP: ${TRAEFIK_IP}
@@ -1,6 +0,0 @@
apiVersion: v1
kind: Namespace
metadata:
name: traefik
labels:
topolvm.io/webhook: ignore
@@ -0,0 +1,3 @@
## DO NOT ALTER THIS FILE, CHANGE DO NOT PERSIST. Alter TalEnv.yaml instead.
serviceMonitor:
enabled: false
@@ -1,67 +1,76 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: descheduler
namespace: kube-system
name: descheduler
namespace: kube-system
spec:
interval: 15m
chart:
spec:
chart: descheduler
version: 0.1.0
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
driftDetection:
mode: warn
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
kind: Deployment
deschedulerPolicy:
strategies:
RemoveDuplicates:
enabled: true
RemovePodsViolatingNodeTaints:
enabled: true
RemovePodsViolatingNodeAffinity:
enabled: true
params:
nodeAffinityType:
- requiredDuringSchedulingIgnoredDuringExecution
RemovePodsViolatingTopologySpreadConstraint:
enabled: true
params:
includeSoftConstraints: true
RemovePodsViolatingInterPodAntiAffinity:
enabled: true
params:
nodeFit: true
LowNodeUtilization:
enabled: false
RemoveFailedPods:
enabled: true
params:
failedPods:
includingInitContainers: true
excludeOwnerKinds:
- Job
minPodLifetimeSeconds: 3600
RemovePodsHavingTooManyRestarts:
enabled: true
params:
podsHavingTooManyRestarts:
podRestartThreshold: 100
includingInitContainers: true
interval: 15m
chart:
spec:
chart: descheduler
version: 0.32.2
sourceRef:
kind: HelmRepository
name: home-ops-mirror
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
replicas: 2
kind: Deployment
deschedulerPolicyAPIVersion: descheduler/v1alpha2
deschedulerPolicy:
profiles:
- name: Default
pluginConfig:
- name: DefaultEvictor
args:
evictFailedBarePods: true
evictLocalStoragePods: true
evictSystemCriticalPods: true
- name: RemoveFailedPods
args:
reasons:
- ContainerStatusUnknown
- NodeAffinity
- NodeShutdown
- Terminated
- UnexpectedAdmissionError
includingInitContainers: true
excludeOwnerKinds:
- Job
minPodLifetimeSeconds: 1800
- name: RemovePodsViolatingInterPodAntiAffinity
- name: RemovePodsViolatingNodeAffinity
args:
nodeAffinityType:
- requiredDuringSchedulingIgnoredDuringExecution
- name: RemovePodsViolatingNodeTaints
- name: RemovePodsViolatingTopologySpreadConstraint
plugins:
balance:
enabled:
- RemovePodsViolatingTopologySpreadConstraint
deschedule:
enabled:
- RemoveFailedPods
- RemovePodsViolatingInterPodAntiAffinity
- RemovePodsViolatingNodeAffinity
- RemovePodsViolatingNodeTaints
service:
enabled: true
serviceMonitor:
enabled: false
leaderElection:
enabled: true
@@ -0,0 +1,85 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: nginx-external
namespace: nginx-external
spec:
interval: 15m
chart:
spec:
chart: 'ingress-nginx'
version: 4.12.1
sourceRef:
kind: HelmRepository
name: 'ingress-nginx'
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
controller:
replicaCount: 2
service:
externalTrafficPolicy: Local
annotations:
# TODO: Maybe play with this
# internal-dns.alpha.kubernetes.io/hostname: external.${DOMAIN_0}
metallb.io/ip-allocated-from-pool: main
metallb.io/loadBalancerIPs: ${NGINX_EXTERNAL_IP}
metallb.universe.tf/ip-allocated-from-pool: main
"lbipam.cilium.io/ips": ${NGINX_EXTERNAL_IP}
ingressClassByName: true
watchIngressWithoutClass: false
ingressClassResource:
name: external
default: false
controllerValue: k8s.io/external
config:
allow-snippet-annotations: true
annotations-risk-level: Critical
client-body-buffer-size: 100M
client-body-timeout: 120
client-header-timeout: 120
enable-brotli: "true"
enable-ocsp: "true"
enable-real-ip: "true"
force-ssl-redirect: "true"
hide-headers: Server,X-Powered-By
hsts-max-age: "31449600"
keep-alive-requests: 10000
keep-alive: 120
proxy-body-size: 0
proxy-buffer-size: 16k
ssl-protocols: TLSv1.3 TLSv1.2
use-forwarded-headers: "true"
metrics:
enabled: true
# serviceMonitor:
# enabled: true
# namespaceSelector:
# any: true
extraArgs:
default-ssl-certificate: "clusterissuer/certificate-issuer-domain-0-wildcard"
publish-status-address: ${NGINX_EXTERNAL_IP}
terminationGracePeriodSeconds: 120
publishService:
enabled: false
resources:
requests:
cpu: 100m
limits:
memory: 500Mi
defaultBackend:
enabled: false
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: nginx-external
@@ -0,0 +1,85 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: nginx-internal
namespace: nginx-internal
spec:
interval: 15m
chart:
spec:
chart: 'ingress-nginx'
version: 4.12.1
sourceRef:
kind: HelmRepository
name: 'ingress-nginx'
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
controller:
replicaCount: 2
service:
externalTrafficPolicy: Local
annotations:
# TODO: Maybe play with this
# internal-dns.alpha.kubernetes.io/hostname: internal.${DOMAIN_0}
metallb.io/ip-allocated-from-pool: main
metallb.io/loadBalancerIPs: ${NGINX_INTERNAL_IP}
metallb.universe.tf/ip-allocated-from-pool: main
"lbipam.cilium.io/ips": ${NGINX_INTERNAL_IP}
ingressClassByName: true
watchIngressWithoutClass: true
ingressClassResource:
name: internal
default: true
controllerValue: k8s.io/internal
config:
allow-snippet-annotations: true
annotations-risk-level: Critical
client-body-buffer-size: 100M
client-body-timeout: 120
client-header-timeout: 120
enable-brotli: "true"
enable-ocsp: "true"
enable-real-ip: "true"
force-ssl-redirect: "true"
hide-headers: Server,X-Powered-By
hsts-max-age: "31449600"
keep-alive-requests: 10000
keep-alive: 120
proxy-body-size: 0
proxy-buffer-size: 16k
ssl-protocols: TLSv1.3 TLSv1.2
use-forwarded-headers: "true"
metrics:
enabled: true
# serviceMonitor:
# enabled: true
# namespaceSelector:
# any: true
extraArgs:
default-ssl-certificate: "clusterissuer/certificate-issuer-domain-0-wildcard"
publish-status-address: ${NGINX_INTERNAL_IP}
terminationGracePeriodSeconds: 120
publishService:
enabled: false
resources:
requests:
cpu: 100m
limits:
memory: 500Mi
defaultBackend:
enabled: false
@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: nginx-internal
@@ -1,26 +1,30 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cert-manager
namespace: cert-manager
name: cert-manager
namespace: cert-manager
spec:
interval: 5m
chart:
spec:
chart: cert-manager
version: 6.5.3
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 5m
install:
createNamespace: true
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
values:
interval: 5m
chart:
spec:
chart: cert-manager
version: v1.17.1
sourceRef:
kind: HelmRepository
name: home-ops-mirror
namespace: flux-system
interval: 5m
install:
createNamespace: true
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
values:
dns01RecursiveNameservers: "1.1.1.1:53,1.0.0.1:53"
dns01RecursiveNameserversOnly: false
installCRDs: true
enableCertificateOwnerRef: true
@@ -1,26 +1,33 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: cloudnative-pg
namespace: cloudnative-pg
name: cloudnative-pg
namespace: cloudnative-pg
spec:
interval: 5m
chart:
spec:
chart: cloudnative-pg
version: 8.2.0
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 5m
install:
createNamespace: true
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
values:
interval: 5m
chart:
spec:
chart: cloudnative-pg
version: 0.23.2
sourceRef:
kind: HelmRepository
name: cloudnative-pg
namespace: flux-system
interval: 5m
install:
createNamespace: true
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
values:
crds:
create: true
replicaCount: 2
# monitoring:
# podMonitorEnabled: false
# grafanaDashboard:
# create: true
@@ -0,0 +1,86 @@
# ---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/monitoring.coreos.com/alertmanagerconfig_v1alpha1.json
# apiVersion: monitoring.coreos.com/v1alpha1
# kind: AlertmanagerConfig
# metadata:
# name: alertmanager
# spec:
# route:
# groupBy: ["alertname", "job"]
# groupInterval: 10m
# groupWait: 1m
# receiver: pushover
# repeatInterval: 12h
# routes:
# - receiver: "null"
# matchers:
# - name: alertname
# value: InfoInhibitor
# matchType: =
# - receiver: heartbeat
# groupInterval: 5m
# groupWait: 0s
# repeatInterval: 5m
# matchers:
# - name: alertname
# value: Watchdog
# matchType: =
# - receiver: pushover
# matchers:
# - name: severity
# value: critical
# matchType: =
# inhibitRules:
# - equal: ["alertname", "namespace"]
# sourceMatch:
# - name: severity
# value: critical
# matchType: =
# targetMatch:
# - name: severity
# value: warning
# matchType: =
# receivers:
# - name: "null"
# - name: heartbeat
# webhookConfigs:
# - urlSecret:
# name: &secret alertmanager-secret
# key: ALERTMANAGER_HEARTBEAT_URL
# - name: pushover
# pushoverConfigs:
# - html: true
# message: |-
# {{- range .Alerts }}
# {{- if ne .Annotations.description "" }}
# {{ .Annotations.description }}
# {{- else if ne .Annotations.summary "" }}
# {{ .Annotations.summary }}
# {{- else if ne .Annotations.message "" }}
# {{ .Annotations.message }}
# {{- else }}
# Alert description not available
# {{- end }}
# {{- if gt (len .Labels.SortedPairs) 0 }}
# <small>
# {{- range .Labels.SortedPairs }}
# <b>{{ .Name }}:</b> {{ .Value }}
# {{- end }}
# </small>
# {{- end }}
# {{- end }}
# priority: |-
# {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
# sendResolved: true
# sound: gamelan
# title: >-
# [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
# {{ .CommonLabels.alertname }}
# ttl: 86400s
# token:
# name: *secret
# key: ALERTMANAGER_PUSHOVER_TOKEN
# userKey:
# name: *secret
# key: PUSHOVER_USER_KEY
# urlTitle: View in Alertmanager
@@ -0,0 +1,112 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: kube-prometheus-stack
namespace: kube-prometheus-stack
spec:
interval: 15m
chart:
spec:
chart: kube-prometheus-stack
version: 69.8.2
sourceRef:
kind: HelmRepository
name: prometheus-community
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values:
crds:
enabled: true
upgradeJob:
enabled: true
forceConflicts: true
cleanPrometheusOperatorObjectNames: true
alertmanager:
enabled: false
kubeEtcd:
service:
selector:
component: kube-apiserver
kubeProxy:
enabled: false
prometheus:
ingress:
enabled: false
prometheusSpec:
disableCompaction: true
podMonitorSelectorNilUsesHelmValues: false
probeSelectorNilUsesHelmValues: false
ruleSelectorNilUsesHelmValues: false
scrapeConfigSelectorNilUsesHelmValues: false
serviceMonitorSelectorNilUsesHelmValues: false
enableAdminAPI: true
walCompression: true
enableFeatures:
- memory-snapshot-on-shutdown
# Prometheus metrics retention time in PV
retention: 24h
retentionSize: 50GB
# External labels to add to any time series or
# alerts when communicating with external systems
externalLabels:
cluster: cluster_main
env: prod
# High availability
replicas: 2
resources:
requests:
cpu: 100m
limits:
memory: 2000Mi
# storageSpec:
# volumeClaimTemplate:
# spec:
# resources:
# requests:
# storage: 60Gi
prometheus-node-exporter:
fullnameOverride: node-exporter
prometheus:
monitor:
enabled: true
relabelings:
- action: replace
regex: (.*)
replacement: $1
sourceLabels: ["__meta_kubernetes_pod_node_name"]
targetLabel: kubernetes_node
kube-state-metrics:
fullnameOverride: kube-state-metrics
# metricLabelsAllowlist:
# - pods=[*]
# - deployments=[*]
# - persistentvolumeclaims=[*]
prometheus:
monitor:
enabled: true
relabelings:
- action: replace
regex: (.*)
replacement: $1
sourceLabels: ["__meta_kubernetes_pod_node_name"]
targetLabel: kubernetes_node
grafana:
enabled: false
forceDeployDashboards: true
defaultDashboardsEnabled: true
forceDeployDatasources: true
@@ -0,0 +1,7 @@
---
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- namespace.yaml
- helm-release.yaml
- alertmanagerconfig.yaml
@@ -0,0 +1,6 @@
apiVersion: v1
kind: Namespace
metadata:
name: kube-prometheus-stack
labels:
pod-security.kubernetes.io/enforce: privileged
@@ -1,26 +0,0 @@
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: prometheus-operator
namespace: prometheus-operator
spec:
interval: 5m
chart:
spec:
chart: prometheus-operator
version: 11.5.1
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 5m
install:
createNamespace: true
crds: CreateReplace
remediation:
retries: 3
upgrade:
crds: CreateReplace
remediation:
retries: 3
values: {}
@@ -10,7 +10,7 @@ spec:
chart:
spec:
chart: app-template
version: 3.7.3
version: 3.5.1
sourceRef:
kind: HelmRepository
name: bjw-s
@@ -31,14 +31,14 @@ spec:
app:
image:
repository: docker.io/rancher/system-upgrade-controller
tag: v0.15.2@sha256:3e899833afcea9a8788d384ce976df9a05be84636fe5c01ec2307b5bd8fe9810
tag: v0.14.1@sha256:7e13a9b2b984f0c0fd6328439b575348723cc6954b91db3453057fcb784e2d29
env:
SYSTEM_UPGRADE_CONTROLLER_DEBUG: false
SYSTEM_UPGRADE_CONTROLLER_THREADS: 2
SYSTEM_UPGRADE_JOB_ACTIVE_DEADLINE_SECONDS: 900
SYSTEM_UPGRADE_JOB_BACKOFF_LIMIT: 99
SYSTEM_UPGRADE_JOB_IMAGE_PULL_POLICY: IfNotPresent
SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: registry.k8s.io/kubectl:v1.32.3
SYSTEM_UPGRADE_JOB_KUBECTL_IMAGE: registry.k8s.io/kubectl:v1.31.1
SYSTEM_UPGRADE_JOB_POD_REPLACEMENT_POLICY: Failed
SYSTEM_UPGRADE_JOB_PRIVILEGED: true
SYSTEM_UPGRADE_JOB_TTL_SECONDS_AFTER_FINISH: 900
@@ -1,10 +1,10 @@
apiVersion: v1
kind: Namespace
metadata:
name: system-upgrade
annotations:
volsync.backube/privileged-movers: "true"
labels:
kustomize.toolkit.fluxcd.io/prune: disabled
goldilocks.fairwinds.com/enabled: "true"
pod-security.kubernetes.io/enforce: privileged
name: system-upgrade
annotations:
volsync.backube/privileged-movers: "true"
labels:
kustomize.toolkit.fluxcd.io/prune: disabled
goldilocks.fairwinds.com/enabled: "true"
pod-security.kubernetes.io/enforce: privileged
@@ -1,32 +0,0 @@
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: traefik-crds
namespace: system
spec:
interval: 15m
chart:
spec:
chart: traefik-crds
version: 3.5.1
sourceRef:
kind: HelmRepository
name: truecharts
namespace: flux-system
interval: 15m
timeout: 20m
maxHistory: 3
driftDetection:
mode: warn
install:
createNamespace: true
remediation:
retries: 3
upgrade:
cleanupOnFail: true
remediation:
retries: 3
uninstall:
keepHistory: false
values: {}
@@ -0,0 +1,11 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/source.toolkit.fluxcd.io/helmrepository_v1.json
apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: home-ops-mirror
namespace: flux-system
spec:
type: oci
interval: 2h
url: oci://ghcr.io/home-operations/charts-mirror/
@@ -10,6 +10,7 @@ resources:
- coredns.yaml
- csi-driver-nfs.yaml
- descheduler.yaml
- home-ops-mirror.yaml
- intel.yaml
- jetstack.yaml
- kubernetes-sigs-metrics-server.yaml