Add kube-state-metrics and CI runner alerts

Deploy kube-state-metrics v2.16.0 in the prometheus namespace with a
read-only ClusterRole limited to the resources it watches, scrape it, and
add an eom-ci alert group: runner not ready, runner container restarted,
runner pod replaced, runner evicted, node DiskPressure, and Gitea down.
The rules pass promtool unit tests, both as written and after rendering
through the Bitnami chart's tpl.

Refs DevOps/software-infrastructure#40

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-10 20:02:16 -05:00
co-authored by Claude Opus 5.5
parent 997b18e7c9
commit bba0d30f53
3 changed files with 179 additions and 0 deletions
+14
View File
@@ -2,6 +2,20 @@
---
# defaults file for ansible-role-prometheus
prometheus_blackbox_exporter_tag: v0.27.0
# kube-state-metrics v2.16 is built against client-go 1.32, within one minor
# version of the cluster's Kubernetes 1.31. Bump together with the cluster.
prometheus_kube_state_metrics_tag: v2.16.0
# Only these resources are watched, which keeps memory and series counts small.
# The ClusterRole in tasks/main.yml grants list/watch on exactly these.
prometheus_kube_state_metrics_resources:
- pods
- nodes
- persistentvolumeclaims
- deployments
- statefulsets
- daemonsets
- jobs
- cronjobs
# Grafana-matrix-forwarder endpoint; the room is the existing Grafana alerts room.
prometheus_matrix_forwarder_url: "http://matrix-forwarder.grafana-matrix-forwarder.svc.cluster.local/api/v1/unified?roomId=!HOBNfTFECIjTOhxwDe:eom.dev"
prometheus_alert_repeat_interval: 24h
+61
View File
@@ -56,3 +56,64 @@ groups:
annotations:
summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'
# CI/CD 0, S7 (DevOps/software-infrastructure#40). Needs kube-state-metrics.
- name: eom-ci
rules:
- alert: GiteaRunnerNotReady
expr: >-
kube_statefulset_replicas{namespace="gitea", statefulset="gitea-actions-runner"}
- kube_statefulset_status_replicas_ready{namespace="gitea", statefulset="gitea-actions-runner"} > 0
for: 10m
labels:
severity: critical
annotations:
summary: 'The Gitea Actions runner has not been ready for 10 minutes'
description: 'No CI jobs can run. Check the pod with kubectl describe and its events (Pending, crash loop, or a missing node label). Runbook: docs/ci-cd/runner.md.'
- alert: GiteaRunnerRestarted
expr: >-
sum by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m])) > 0
labels:
severity: warning
annotations:
summary: 'Runner container {{`{{ $labels.container }}`}} restarted'
description: 'Jobs that were running on the runner were lost. Check the previous container log with kubectl logs --previous.'
- alert: GiteaRunnerPodReplaced
# A replaced pod keeps its name but gets a new uid, so it shows up as a
# second kube_pod_created series for the same pod name within the window.
expr: >-
count by (namespace, pod) (max_over_time(kube_pod_created{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m])) > 1
or
changes(kube_pod_created{namespace="gitea", pod=~"gitea-actions-runner-.*"}[15m]) > 0
labels:
severity: warning
annotations:
summary: 'The runner pod {{`{{ $labels.pod }}`}} was replaced'
description: 'The pod was deleted, evicted or rescheduled, and any running jobs were lost. Expected after a runner deploy; otherwise check the events and GiteaRunnerEvicted.'
- alert: GiteaRunnerEvicted
expr: >-
max by (namespace, pod) (max_over_time(kube_pod_status_reason{namespace="gitea", pod=~"gitea-actions-runner-.*", reason="Evicted"}[15m])) > 0
labels:
severity: critical
annotations:
summary: 'The runner pod {{`{{ $labels.pod }}`}} was evicted'
description: 'Usually the node ran short of disk or memory. Running jobs died without a step error. Check the node conditions and the runner Docker storage.'
- alert: NodeDiskPressure
expr: kube_node_status_condition{condition="DiskPressure", status="true"} == 1
for: 5m
labels:
severity: warning
annotations:
summary: '{{`{{ $labels.node }}`}} reports DiskPressure'
description: 'The kubelet will evict pods from this node until disk usage drops. Free space or move workloads.'
- alert: GiteaDown
expr: >-
kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"} < 1
or
absent(kube_deployment_status_replicas_available{namespace="gitea", deployment="gitea"})
for: 5m
labels:
severity: critical
annotations:
summary: 'Gitea has no available pod'
description: 'Git, the web UI, the API and CI are all down. SiteDown covers the public URL; this alert covers the pod itself.'
+104
View File
@@ -45,6 +45,102 @@
- port: 9115
targetPort: 9115
- name: Deploy kube-state-metrics (pod, node and workload state for alerts)
kubernetes.core.k8s:
state: present
definition:
- apiVersion: v1
kind: ServiceAccount
metadata:
name: kube-state-metrics
namespace: prometheus
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: kube-state-metrics
rules:
# Read-only, and limited to the resources passed to --resources below.
- apiGroups: [""]
resources: [pods, nodes, persistentvolumeclaims]
verbs: [list, watch]
- apiGroups: [apps]
resources: [deployments, statefulsets, daemonsets]
verbs: [list, watch]
- apiGroups: [batch]
resources: [jobs, cronjobs]
verbs: [list, watch]
- apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kube-state-metrics
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kube-state-metrics
subjects:
- kind: ServiceAccount
name: kube-state-metrics
namespace: prometheus
- apiVersion: apps/v1
kind: Deployment
metadata:
name: kube-state-metrics
namespace: prometheus
labels:
app: kube-state-metrics
spec:
replicas: 1
selector:
matchLabels:
app: kube-state-metrics
template:
metadata:
labels:
app: kube-state-metrics
spec:
serviceAccountName: kube-state-metrics
containers:
- name: kube-state-metrics
image: "registry.k8s.io/kube-state-metrics/kube-state-metrics:{{ prometheus_kube_state_metrics_tag }}"
args:
- "--resources={{ prometheus_kube_state_metrics_resources | join(',') }}"
ports:
- name: http-metrics
containerPort: 8080
readinessProbe:
httpGet:
path: /readyz
port: 8081
livenessProbe:
httpGet:
path: /livez
port: 8080
securityContext:
runAsNonRoot: true
runAsUser: 65534
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
capabilities:
drop: [ALL]
resources:
requests:
cpu: 10m
memory: 64Mi
limits:
memory: 256Mi
- apiVersion: v1
kind: Service
metadata:
name: kube-state-metrics
namespace: prometheus
spec:
selector:
app: kube-state-metrics
ports:
- name: http-metrics
port: 8080
targetPort: 8080
- name: Deploy Prometheus
kubernetes.core.helm:
name: prometheus
@@ -224,6 +320,14 @@
- targets:
- nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400
# Pod, node and workload state, used by the eom-ci alerts.
# honor_labels keeps the namespace and pod labels kube-state-metrics sets.
- job_name: kube-state-metrics
honor_labels: true
static_configs:
- targets:
- kube-state-metrics.prometheus.svc.cluster.local:8080
- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix)
kubernetes.core.k8s:
state: present