Alerting to Matrix, site probes, GPU metrics and a forum digest

- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
  CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
  HostOverheating and HostRebooted. Disk and RAM are already covered by
  Grafana rules. Templates use Helm's {{` `}} escaping because the chart
  runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
  Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
  plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
  discourse.eom.dev to Matrix through the same forwarder, and nothing when
  there are none. Standard-library Python, public Discourse JSON only.

These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

files/alerting-rules.yml

@@ -0,0 +1,58 @@
+# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml.
+# The chart passes alertingRules through Helm's tpl, so every Prometheus template
+# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this
+# file with lookup('file'), which it does not template.
+# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here.
+groups:
+  - name: eom-availability
+    rules:
+      - alert: SiteDown
+        expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500)
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} is down'
+          description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).'
+      - alert: CertificateExpiringSoon
+        expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days'
+          description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.'
+      - alert: ScrapeTargetDown
+        expr: up == 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})'
+          description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.'
+
+  - name: eom-hardware
+    rules:
+      - alert: GpuOverheating
+        expr: DCGM_FI_DEV_GPU_TEMP >= 85
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C'
+          description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.'
+      - alert: HostOverheating
+        expr: max by (instance) (node_hwmon_temp_celsius) >= 85
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C'
+          description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.'
+      - alert: HostRebooted
+        expr: time() - node_boot_time_seconds < 900
+        labels:
+          severity: info
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
+          description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'

files/forum-digest.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Daily digest of unanswered discourse.eom.dev topics, posted to Matrix.
+
+Lists recent public topics that have no replies yet, skipping pinned, closed
+and archived topics, and topics started by staff or EXCLUDE_AUTHORS (by default
+Eric and the assistant). Posts one message through grafana-matrix-forwarder (Alertmanager webhook format), and
+nothing when every topic has an answer. Standard library only.
+"""
+import datetime
+import hashlib
+import json
+import os
+import sys
+import urllib.request
+
+FORUM = os.environ.get("FORUM_URL", "https://discourse.eom.dev").rstrip("/")
+FORWARDER = os.environ["FORWARDER_URL"]
+MAX_AGE_DAYS = int(os.environ.get("MAX_AGE_DAYS", "30"))
+DRY_RUN = os.environ.get("DRY_RUN") == "1"
+# Authors whose topics never need a reply from Eric (his own posts and the assistant's).
+EXCLUDE = {u.strip() for u in os.environ.get("EXCLUDE_AUTHORS", "eric,claude").split(",") if u.strip()}
+
+
+def get(path):
+    req = urllib.request.Request(FORUM + path, headers={"Accept": "application/json", "User-Agent": "eom-forum-digest"})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.load(r)
+
+
+def staff_usernames():
+    about = get("/about.json").get("about", {})
+    return {u["username"] for key in ("admins", "moderators") for u in about.get(key, [])}
+
+
+def unanswered():
+    staff = staff_usernames()
+    data = get("/latest.json?order=created")
+    users = {u["id"]: u["username"] for u in data.get("users", [])}
+    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=MAX_AGE_DAYS)
+    out = []
+    for t in data.get("topic_list", {}).get("topics", []):
+        if t.get("posts_count", 0) != 1 or t.get("pinned") or t.get("closed") or t.get("archived"):
+            continue
+        created = datetime.datetime.fromisoformat(t["created_at"].replace("Z", "+00:00"))
+        if created < cutoff:
+            continue
+        op = next((users.get(p["user_id"]) for p in t.get("posters", []) if "Original Poster" in (p.get("description") or "")), None)
+        if op in staff or op in EXCLUDE:
+            continue
+        age = (datetime.datetime.now(datetime.timezone.utc) - created).days
+        out.append({"id": t["id"], "title": t["title"], "author": op or "?", "age_days": age,
+                    "url": f"{FORUM}/t/{t['slug']}/{t['id']}"})
+    return sorted(out, key=lambda x: -x["age_days"])
+
+
+def post(topics):
+    today = datetime.date.today().isoformat()
+    lines = [f"- {t['title']} by @{t['author']}, {t['age_days']}d old: {t['url']}" for t in topics]
+    summary = f"{len(topics)} forum topic{'s' if len(topics) != 1 else ''} waiting for a reply:\n" + "\n".join(lines)
+    alert = {
+        "status": "firing",
+        "labels": {"alertname": "ForumUnanswered", "severity": "info", "instance": "discourse.eom.dev"},
+        "annotations": {"summary": summary, "description": "Daily forum digest (eom-forum-digest CronJob)."},
+        "startsAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        "endsAt": "0001-01-01T00:00:00Z",
+        "generatorURL": FORUM + "/latest",
+        # One fingerprint per day: a new message each day rather than a reply thread.
+        "fingerprint": hashlib.sha1(f"forum-digest-{today}".encode()).hexdigest()[:16],
+    }
+    payload = {
+        "version": "4", "receiver": "matrix", "status": "firing", "groupKey": f"forum-digest-{today}",
+        "externalURL": FORUM, "groupLabels": {"alertname": "ForumUnanswered"},
+        "commonLabels": alert["labels"], "commonAnnotations": alert["annotations"], "alerts": [alert],
+    }
+    if DRY_RUN:
+        print(json.dumps(payload, indent=2))
+        return
+    req = urllib.request.Request(FORWARDER, data=json.dumps(payload).encode(), method="POST",
+                                 headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        print("forwarder:", r.status)
+
+
+def main():
+    topics = unanswered()
+    print(f"{len(topics)} unanswered topic(s)")
+    for t in topics:
+        print(f"  {t['id']} {t['title']} ({t['age_days']}d, @{t['author']})")
+    if topics:
+        post(topics)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())