claudeandClaude Opus 5.5
997b18e7c9
- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
HostOverheating and HostRebooted. Disk and RAM are already covered by
Grafana rules. Templates use Helm's {{` `}} escaping because the chart
runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
discourse.eom.dev to Matrix through the same forwarder, and nothing when
there are none. Standard-library Python, public Discourse JSON only.
These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
59 lines
2.9 KiB
YAML
59 lines
2.9 KiB
YAML
# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml.
|
||
# The chart passes alertingRules through Helm's tpl, so every Prometheus template
|
||
# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this
|
||
# file with lookup('file'), which it does not template.
|
||
# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here.
|
||
groups:
|
||
- name: eom-availability
|
||
rules:
|
||
- alert: SiteDown
|
||
expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500)
|
||
for: 5m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: '{{`{{ $labels.instance }}`}} is down'
|
||
description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).'
|
||
- alert: CertificateExpiringSoon
|
||
expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14
|
||
for: 1h
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days'
|
||
description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.'
|
||
- alert: ScrapeTargetDown
|
||
expr: up == 0
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})'
|
||
description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.'
|
||
|
||
- name: eom-hardware
|
||
rules:
|
||
- alert: GpuOverheating
|
||
expr: DCGM_FI_DEV_GPU_TEMP >= 85
|
||
for: 10m
|
||
labels:
|
||
severity: critical
|
||
annotations:
|
||
summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C'
|
||
description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.'
|
||
- alert: HostOverheating
|
||
expr: max by (instance) (node_hwmon_temp_celsius) >= 85
|
||
for: 15m
|
||
labels:
|
||
severity: warning
|
||
annotations:
|
||
summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C'
|
||
description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.'
|
||
- alert: HostRebooted
|
||
expr: time() - node_boot_time_seconds < 900
|
||
labels:
|
||
severity: info
|
||
annotations:
|
||
summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
|
||
description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'
|