Alerting to Matrix, site probes, GPU metrics and a forum digest

- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
  CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
  HostOverheating and HostRebooted. Disk and RAM are already covered by
  Grafana rules. Templates use Helm's {{` `}} escaping because the chart
  runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
  Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
  plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
  discourse.eom.dev to Matrix through the same forwarder, and nothing when
  there are none. Standard-library Python, public Discourse JSON only.

These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

defaults/main.yml

@@ -1,3 +1,34 @@
 #SPDX-License-Identifier: MIT-0
 ---
 # defaults file for ansible-role-prometheus
+prometheus_blackbox_exporter_tag: v0.27.0
+# Grafana-matrix-forwarder endpoint; the room is the existing Grafana alerts room.
+prometheus_matrix_forwarder_url: "http://matrix-forwarder.grafana-matrix-forwarder.svc.cluster.local/api/v1/unified?roomId=!HOBNfTFECIjTOhxwDe:eom.dev"
+prometheus_alert_repeat_interval: 24h
+prometheus_blackbox_targets:
+  - https://www.eom.dev
+  - https://eom.dev
+  - https://discourse.eom.dev
+  - https://gitea.eom.dev
+  - https://nextcloud.eom.dev
+  - https://element.eom.dev
+  - https://synapse.eom.dev
+  - https://mas.eom.dev
+  - https://dex.eom.dev
+  - https://peertube.eom.dev
+  - https://owncast.eom.dev
+  - https://mastodon.eom.dev
+  - https://listmonk.eom.dev
+  - https://mailman.eom.dev
+  - https://grafana.eom.dev
+  - https://localai.eom.dev
+  - https://wanderer.eom.dev
+  - https://wikideck.eom.dev
+  - https://button.eom.dev
+  - https://collabora.eom.dev
+  - https://fediprofile.eom.dev
+  - https://minio.eom.dev
+  - https://shkeeper.eom.dev
+# Daily digest of unanswered forum topics (9:00 US Eastern).
+prometheus_forum_digest_schedule: "0 9 * * *"
+prometheus_forum_digest_exclude: [eric, claude]

files/alerting-rules.yml

@@ -0,0 +1,58 @@
+# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml.
+# The chart passes alertingRules through Helm's tpl, so every Prometheus template
+# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this
+# file with lookup('file'), which it does not template.
+# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here.
+groups:
+  - name: eom-availability
+    rules:
+      - alert: SiteDown
+        expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500)
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} is down'
+          description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).'
+      - alert: CertificateExpiringSoon
+        expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14
+        for: 1h
+        labels:
+          severity: warning
+        annotations:
+          summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days'
+          description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.'
+      - alert: ScrapeTargetDown
+        expr: up == 0
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})'
+          description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.'
+
+  - name: eom-hardware
+    rules:
+      - alert: GpuOverheating
+        expr: DCGM_FI_DEV_GPU_TEMP >= 85
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C'
+          description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.'
+      - alert: HostOverheating
+        expr: max by (instance) (node_hwmon_temp_celsius) >= 85
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C'
+          description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.'
+      - alert: HostRebooted
+        expr: time() - node_boot_time_seconds < 900
+        labels:
+          severity: info
+        annotations:
+          summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
+          description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'

files/forum-digest.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Daily digest of unanswered discourse.eom.dev topics, posted to Matrix.
+
+Lists recent public topics that have no replies yet, skipping pinned, closed
+and archived topics, and topics started by staff or EXCLUDE_AUTHORS (by default
+Eric and the assistant). Posts one message through grafana-matrix-forwarder (Alertmanager webhook format), and
+nothing when every topic has an answer. Standard library only.
+"""
+import datetime
+import hashlib
+import json
+import os
+import sys
+import urllib.request
+
+FORUM = os.environ.get("FORUM_URL", "https://discourse.eom.dev").rstrip("/")
+FORWARDER = os.environ["FORWARDER_URL"]
+MAX_AGE_DAYS = int(os.environ.get("MAX_AGE_DAYS", "30"))
+DRY_RUN = os.environ.get("DRY_RUN") == "1"
+# Authors whose topics never need a reply from Eric (his own posts and the assistant's).
+EXCLUDE = {u.strip() for u in os.environ.get("EXCLUDE_AUTHORS", "eric,claude").split(",") if u.strip()}
+
+
+def get(path):
+    req = urllib.request.Request(FORUM + path, headers={"Accept": "application/json", "User-Agent": "eom-forum-digest"})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        return json.load(r)
+
+
+def staff_usernames():
+    about = get("/about.json").get("about", {})
+    return {u["username"] for key in ("admins", "moderators") for u in about.get(key, [])}
+
+
+def unanswered():
+    staff = staff_usernames()
+    data = get("/latest.json?order=created")
+    users = {u["id"]: u["username"] for u in data.get("users", [])}
+    cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=MAX_AGE_DAYS)
+    out = []
+    for t in data.get("topic_list", {}).get("topics", []):
+        if t.get("posts_count", 0) != 1 or t.get("pinned") or t.get("closed") or t.get("archived"):
+            continue
+        created = datetime.datetime.fromisoformat(t["created_at"].replace("Z", "+00:00"))
+        if created < cutoff:
+            continue
+        op = next((users.get(p["user_id"]) for p in t.get("posters", []) if "Original Poster" in (p.get("description") or "")), None)
+        if op in staff or op in EXCLUDE:
+            continue
+        age = (datetime.datetime.now(datetime.timezone.utc) - created).days
+        out.append({"id": t["id"], "title": t["title"], "author": op or "?", "age_days": age,
+                    "url": f"{FORUM}/t/{t['slug']}/{t['id']}"})
+    return sorted(out, key=lambda x: -x["age_days"])
+
+
+def post(topics):
+    today = datetime.date.today().isoformat()
+    lines = [f"- {t['title']} by @{t['author']}, {t['age_days']}d old: {t['url']}" for t in topics]
+    summary = f"{len(topics)} forum topic{'s' if len(topics) != 1 else ''} waiting for a reply:\n" + "\n".join(lines)
+    alert = {
+        "status": "firing",
+        "labels": {"alertname": "ForumUnanswered", "severity": "info", "instance": "discourse.eom.dev"},
+        "annotations": {"summary": summary, "description": "Daily forum digest (eom-forum-digest CronJob)."},
+        "startsAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
+        "endsAt": "0001-01-01T00:00:00Z",
+        "generatorURL": FORUM + "/latest",
+        # One fingerprint per day: a new message each day rather than a reply thread.
+        "fingerprint": hashlib.sha1(f"forum-digest-{today}".encode()).hexdigest()[:16],
+    }
+    payload = {
+        "version": "4", "receiver": "matrix", "status": "firing", "groupKey": f"forum-digest-{today}",
+        "externalURL": FORUM, "groupLabels": {"alertname": "ForumUnanswered"},
+        "commonLabels": alert["labels"], "commonAnnotations": alert["annotations"], "alerts": [alert],
+    }
+    if DRY_RUN:
+        print(json.dumps(payload, indent=2))
+        return
+    req = urllib.request.Request(FORWARDER, data=json.dumps(payload).encode(), method="POST",
+                                 headers={"Content-Type": "application/json"})
+    with urllib.request.urlopen(req, timeout=30) as r:
+        print("forwarder:", r.status)
+
+
+def main():
+    topics = unanswered()
+    print(f"{len(topics)} unanswered topic(s)")
+    for t in topics:
+        print(f"  {t['id']} {t['title']} ({t['age_days']}d, @{t['author']})")
+    if topics:
+        post(topics)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tasks/main.yml

@@ -1,6 +1,50 @@
 #SPDX-License-Identifier: MIT-0
 ---
 # tasks file for ansible-role-prometheus
+- name: Deploy blackbox exporter (HTTPS probes of public sites)
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      - apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: blackbox-exporter
+          namespace: prometheus
+          labels:
+            app: blackbox-exporter
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: blackbox-exporter
+          template:
+            metadata:
+              labels:
+                app: blackbox-exporter
+            spec:
+              containers:
+                - name: blackbox-exporter
+                  image: "prom/blackbox-exporter:{{ prometheus_blackbox_exporter_tag }}"
+                  ports:
+                    - containerPort: 9115
+                  resources:
+                    requests:
+                      cpu: 10m
+                      memory: 32Mi
+                    limits:
+                      memory: 128Mi
+      - apiVersion: v1
+        kind: Service
+        metadata:
+          name: blackbox-exporter
+          namespace: prometheus
+        spec:
+          selector:
+            app: blackbox-exporter
+          ports:
+            - port: 9115
+              targetPort: 9115
+
 - name: Deploy Prometheus
   kubernetes.core.helm:
     name: prometheus
@@ -12,6 +56,18 @@
         image: 
           repository: bitnamilegacy/alertmanager
           tag: 0.28.1-debian-12-r15
+        configuration: |
+          route:
+            receiver: matrix
+            group_by: [alertname, instance]
+            group_wait: 30s
+            group_interval: 5m
+            repeat_interval: {{ prometheus_alert_repeat_interval }}
+          receivers:
+            - name: matrix
+              webhook_configs:
+                - url: "{{ prometheus_matrix_forwarder_url }}"
+                  send_resolved: true
       server:
         image: 
           repository: bitnamilegacy/prometheus
@@ -26,6 +82,7 @@
             memory: 2Gi
         persistence:
           size: 256Gi
+        alertingRules: "{{ lookup('ansible.builtin.file', 'alerting-rules.yml') | from_yaml }}"
         extraScrapeConfigs:
           - job_name: local
             static_configs:
@@ -147,3 +204,74 @@
                 labels:
                   instance: alpha-worker-17
             metrics_path: /metrics
+
+          - job_name: blackbox
+            metrics_path: /probe
+            params:
+              module: [http_2xx]
+            static_configs:
+              - targets: "{{ prometheus_blackbox_targets }}"
+            relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: blackbox-exporter.prometheus.svc.cluster.local:9115
+
+          - job_name: dcgm
+            static_configs:
+              - targets:
+                - nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400
+
+- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix)
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      - apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: forum-digest
+          namespace: prometheus
+        data:
+          forum-digest.py: "{{ lookup('ansible.builtin.file', 'forum-digest.py') }}"
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: forum-digest
+          namespace: prometheus
+        spec:
+          schedule: "{{ prometheus_forum_digest_schedule }}"
+          timeZone: America/New_York
+          concurrencyPolicy: Forbid
+          successfulJobsHistoryLimit: 3
+          failedJobsHistoryLimit: 3
+          jobTemplate:
+            spec:
+              backoffLimit: 1
+              ttlSecondsAfterFinished: 86400
+              template:
+                spec:
+                  restartPolicy: Never
+                  containers:
+                    - name: forum-digest
+                      image: python:3.13-alpine
+                      command: ["python3", "-I", "/app/forum-digest.py"]
+                      env:
+                        - name: FORWARDER_URL
+                          value: "{{ prometheus_matrix_forwarder_url }}"
+                        - name: EXCLUDE_AUTHORS
+                          value: "{{ prometheus_forum_digest_exclude | join(',') }}"
+                      resources:
+                        requests:
+                          cpu: 10m
+                          memory: 32Mi
+                        limits:
+                          memory: 128Mi
+                      volumeMounts:
+                        - name: app
+                          mountPath: /app
+                  volumes:
+                    - name: app
+                      configMap:
+                        name: forum-digest