Add kube-state-metrics and CI runner alerts
Deploy kube-state-metrics v2.16.0 in the prometheus namespace with a read-only ClusterRole limited to the resources it watches, scrape it, and add an eom-ci alert group: runner not ready, runner container restarted, runner pod replaced, runner evicted, node DiskPressure, and Gitea down. The rules pass promtool unit tests, both as written and after rendering through the Bitnami chart's tpl. Refs DevOps/software-infrastructure#40 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,6 +2,20 @@
|
||||
---
|
||||
# defaults file for ansible-role-prometheus
|
||||
prometheus_blackbox_exporter_tag: v0.27.0
|
||||
# kube-state-metrics v2.16 is built against client-go 1.32, within one minor
|
||||
# version of the cluster's Kubernetes 1.31. Bump together with the cluster.
|
||||
prometheus_kube_state_metrics_tag: v2.16.0
|
||||
# Only these resources are watched, which keeps memory and series counts small.
|
||||
# The ClusterRole in tasks/main.yml grants list/watch on exactly these.
|
||||
prometheus_kube_state_metrics_resources:
|
||||
- pods
|
||||
- nodes
|
||||
- persistentvolumeclaims
|
||||
- deployments
|
||||
- statefulsets
|
||||
- daemonsets
|
||||
- jobs
|
||||
- cronjobs
|
||||
# Grafana-matrix-forwarder endpoint; the room is the existing Grafana alerts room.
|
||||
prometheus_matrix_forwarder_url: "http://matrix-forwarder.grafana-matrix-forwarder.svc.cluster.local/api/v1/unified?roomId=!HOBNfTFECIjTOhxwDe:eom.dev"
|
||||
prometheus_alert_repeat_interval: 24h
|
||||
|
||||
@@ -56,3 +56,64 @@ groups:
|
||||
annotations:
|
||||
summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
|
||||
description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'
|
||||
|
||||
# CI/CD 0, S7 (DevOps/software-infrastructure#40). Needs kube-state-metrics.
|
||||
- name: eom-ci
|
||||
rules:
|
||||
- alert: GiteaRunnerNotReady
|
||||
expr: >-
|
||||
kube_statefulset_replicas{namespace="gitea", statefulset="gitea-actions-runner"}
|
||||
- kube_statefulset_status_replicas_ready{namespace="gitea", statefulset="gitea-actions-runner"} > 0
|
||||
for: 10m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 'The Gitea Actions runner has not been ready for 10 minutes'
|
||||
description: 'No CI jobs can run. Check the pod with kubectl describe and its events (Pending, crash loop, or a missing node label). Runbook: docs/ci-cd/runner.md.'
|
||||
- alert: GiteaRunnerRestarted
|
||||
expr: >-
|
||||
sum by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m])) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 'Runner container {{`{{ $labels.container }}`}} restarted'
|
||||
description: 'Jobs that were running on the runner were lost. Check the previous container log with kubectl logs --previous.'
|
||||
- alert: GiteaRunnerPodReplaced
|
||||
# A replaced pod keeps its name but gets a new uid, so it shows up as a
|
||||
# second kube_pod_created series for the same pod name within the window.
|
||||
expr: >-
|
||||
count by (namespace, pod) (max_over_time(kube_pod_created{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m])) > 1
|
||||
or
|
||||
changes(kube_pod_created{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m]) > 0
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: 'The runner pod {{`{{ $labels.pod }}`}} was replaced'
|
||||
description: 'The pod was deleted, evicted or rescheduled, and any running jobs were lost. Expected after a runner deploy; otherwise check the events and GiteaRunnerEvicted.'
|
||||
- alert: GiteaRunnerEvicted
|
||||
expr: >-
|
||||
max by (namespace, pod) (max_over_time(kube_pod_status_reason{namespace="gitea", pod=~"gitea-actions-runner-.*", reason="Evicted"}[15m])) > 0
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 'The runner pod {{`{{ $labels.pod }}`}} was evicted'
|
||||
description: 'Usually the node ran short of disk or memory. Running jobs died without a step error. Check the node conditions and the runner Docker storage.'
|
||||
- alert: NodeDiskPressure
|
||||
expr: kube_node_status_condition{condition="DiskPressure", status="true"} == 1
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: '{{`{{ $labels.node }}`}} reports DiskPressure'
|
||||
description: 'The kubelet will evict pods from this node until disk usage drops. Free space or move workloads.'
|
||||
- alert: GiteaDown
|
||||
expr: >-
|
||||
kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"} < 1
|
||||
or
|
||||
absent(kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"})
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: 'Gitea has no available pod'
|
||||
description: 'Git, the web UI, the API and CI are all down. SiteDown covers the public URL; this alert covers the pod itself.'
|
||||
|
||||
+104
@@ -45,6 +45,102 @@
|
||||
- port: 9115
|
||||
targetPort: 9115
|
||||
|
||||
- name: Deploy kube-state-metrics (pod, node and workload state for alerts)
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
definition:
|
||||
- apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: prometheus
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRole
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
rules:
|
||||
# Read-only, and limited to the resources passed to --resources below.
|
||||
- apiGroups: [""]
|
||||
resources: [pods, nodes, persistentvolumeclaims]
|
||||
verbs: [list, watch]
|
||||
- apiGroups: [apps]
|
||||
resources: [deployments, statefulsets, daemonsets]
|
||||
verbs: [list, watch]
|
||||
- apiGroups: [batch]
|
||||
resources: [jobs, cronjobs]
|
||||
verbs: [list, watch]
|
||||
- apiVersion: rbac.authorization.k8s.io/v1
|
||||
kind: ClusterRoleBinding
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
roleRef:
|
||||
apiGroup: rbac.authorization.k8s.io
|
||||
kind: ClusterRole
|
||||
name: kube-state-metrics
|
||||
subjects:
|
||||
- kind: ServiceAccount
|
||||
name: kube-state-metrics
|
||||
namespace: prometheus
|
||||
- apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: prometheus
|
||||
labels:
|
||||
app: kube-state-metrics
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app: kube-state-metrics
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: kube-state-metrics
|
||||
spec:
|
||||
serviceAccountName: kube-state-metrics
|
||||
containers:
|
||||
- name: kube-state-metrics
|
||||
image: "registry.k8s.io/kube-state-metrics/kube-state-metrics:{{ prometheus_kube_state_metrics_tag }}"
|
||||
args:
|
||||
- "--resources={{ prometheus_kube_state_metrics_resources | join(',') }}"
|
||||
ports:
|
||||
- name: http-metrics
|
||||
containerPort: 8080
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /readyz
|
||||
port: 8081
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /livez
|
||||
port: 8080
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 64Mi
|
||||
limits:
|
||||
memory: 256Mi
|
||||
- apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: kube-state-metrics
|
||||
namespace: prometheus
|
||||
spec:
|
||||
selector:
|
||||
app: kube-state-metrics
|
||||
ports:
|
||||
- name: http-metrics
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
|
||||
- name: Deploy Prometheus
|
||||
kubernetes.core.helm:
|
||||
name: prometheus
|
||||
@@ -224,6 +320,14 @@
|
||||
- targets:
|
||||
- nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400
|
||||
|
||||
# Pod, node and workload state, used by the eom-ci alerts.
|
||||
# honor_labels keeps the namespace and pod labels kube-state-metrics sets.
|
||||
- job_name: kube-state-metrics
|
||||
honor_labels: true
|
||||
static_configs:
|
||||
- targets:
|
||||
- kube-state-metrics.prometheus.svc.cluster.local:8080
|
||||
|
||||
- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix)
|
||||
kubernetes.core.k8s:
|
||||
state: present
|
||||
|
||||
Reference in New Issue
Block a user