From 997b18e7c9b6d15c6117f9877158a9b99bf55c92 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 7 Nov 2026 16:20:42 -0500 Subject: [PATCH] Alerting to Matrix, site probes, GPU metrics and a forum digest - Prometheus alert rules (files/alerting-rules.yml): SiteDown, CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C), HostOverheating and HostRebooted. Disk and RAM are already covered by Grafana rules. Templates use Helm's {{` `}} escaping because the chart runs alertingRules through tpl. - Alertmanager routes everything to the grafana-matrix-forwarder (same Matrix room as Grafana alerts), resolved notifications on, repeat 24h. - Blackbox exporter (prom/blackbox-exporter) probing every public site, plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks). - forum-digest CronJob (9:00 ET): posts unanswered member topics on discourse.eom.dev to Matrix through the same forwarder, and nothing when there are none. Standard-library Python, public Discourse JSON only. These replace the LocalAI SRE and community agents, which were too slow and unreliable on the throttled T4. Co-Authored-By: Claude Opus 5.5 --- defaults/main.yml | 31 +++++ .../__pycache__/forum-digest.cpython-313.pyc | Bin 0 -> 7984 bytes files/alerting-rules.yml | 58 ++++++++ files/forum-digest.py | 95 +++++++++++++ tasks/main.yml | 128 ++++++++++++++++++ 5 files changed, 312 insertions(+) create mode 100644 files/__pycache__/forum-digest.cpython-313.pyc create mode 100644 files/alerting-rules.yml create mode 100644 files/forum-digest.py diff --git a/defaults/main.yml b/defaults/main.yml index 1564c2b..5523a44 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,3 +1,34 @@ #SPDX-License-Identifier: MIT-0 --- # defaults file for ansible-role-prometheus +prometheus_blackbox_exporter_tag: v0.27.0 +# Grafana-matrix-forwarder endpoint; the room is the existing Grafana alerts room. +prometheus_matrix_forwarder_url: "http://matrix-forwarder.grafana-matrix-forwarder.svc.cluster.local/api/v1/unified?roomId=!HOBNfTFECIjTOhxwDe:eom.dev" +prometheus_alert_repeat_interval: 24h +prometheus_blackbox_targets: + - https://www.eom.dev + - https://eom.dev + - https://discourse.eom.dev + - https://gitea.eom.dev + - https://nextcloud.eom.dev + - https://element.eom.dev + - https://synapse.eom.dev + - https://mas.eom.dev + - https://dex.eom.dev + - https://peertube.eom.dev + - https://owncast.eom.dev + - https://mastodon.eom.dev + - https://listmonk.eom.dev + - https://mailman.eom.dev + - https://grafana.eom.dev + - https://localai.eom.dev + - https://wanderer.eom.dev + - https://wikideck.eom.dev + - https://button.eom.dev + - https://collabora.eom.dev + - https://fediprofile.eom.dev + - https://minio.eom.dev + - https://shkeeper.eom.dev +# Daily digest of unanswered forum topics (9:00 US Eastern). +prometheus_forum_digest_schedule: "0 9 * * *" +prometheus_forum_digest_exclude: [eric, claude] diff --git a/files/__pycache__/forum-digest.cpython-313.pyc b/files/__pycache__/forum-digest.cpython-313.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1d8a086268b8427c1a46316c5b2fdcc8f1484afa GIT binary patch literal 7984 zcmb_hYit`=cAgmy-!F@zEkEV49sCBH&jiWFItV~(iEmK=6OjwssX4EGLc zTii6O-TkrBI<4deL1YxzC_sKl*aA`71y%tHtb%P3_fIindP`-j9|5{pz`yE8fov8n z&~t|zN|u$RTl7M@_s+fNzRtP#obMbyb2@DVr1#(Zk89F4g7|m*p(j(h^6U)?CTikLgo=V46|-d;p)vT1i>0u`WUQ(Q+Ru++?cb* zV`S+Y*|(mRH5q{bIYyjy#MTsBu4`s3`nr`BTz|%Rt3Hagb@agc?muI!optDIhYxH# zZK=*W^|`BwK|-S1;J=#a@ezbbgzI0VKUQu!R4(Jd&(JP{hR>;i#2 ztZO%ctgIXIYIYmsUbYJI?W_m#9V0btb;l4va61WN#29XUaTmb-9d(4h7ySebk)RTk zM3q(alR>q|;mi9vjf7qUzyH=0Vu%H<oXaxf+zp-)M4-r|lDx!B z@Y!ZYx}MBrl0t%kOyJ|qOe`fzeB8zfafU;&@#GC{mC~ZPv9tk3%5n&AU~~dk$Ho{D zFdZnW{^Ji|#6?3EL;z5w$E97JM7foIaZ zBykBIHjBhuVw_1JZj2K+e_A`Be@sL-ITYs+b10bNQ8o>I3HZ9nkB*Dtbq4x@($|bv zu?b>!9G~juI4>~#4IWLDHV#aXfB^n5hXV`?yNJWAOe#5wIG_@R)I`Ac2#d5|wZ3*S z90^4u;lBK?@oY9Db+oj+AQF{qQJp~eW-#2@6~@G>BNQBp2D`hWox#hjN*!rc&7I-P z(Qu?+CEHZ@CO+luJTRv@mf~`8UL~7jT zZ%V_u3WF!$hd!*9SVMqf>2^vWzh^${rpe$lLb68I)J{=^Kn-h(6VrqpFH+VK5lX-M z62*e@WC{JLGeiS%fM_Nv&lJu)1`lIDJVAIS#A(BRVvO{e`~S3!%fNMwaoMCOv|N)! zA-@gY0=^Bow7>Lde3WVm#$tRXo3}+I9{Ga_cmh6BHIMTgTo~z5E$nU)zO%_RFXpm7 zvuetr6v$jPBmM_DczCKgT&@^3p{r&f5Hq}>(s)soPKjLHXHe-3mmOCPFiItngeAi; zjJka+nc}6EjhhN&CR9U$&mz1wC<;Ovglv*nvu&FROb3cKMzJv;Z(s0!v{ybiAjfYg z$8X9uMz-CWJilgj&(u!W7Ondg>;8GeLRiMN)}`Q@yLx8g=M!^Px8MHSK-k)f)>ioY zlZ3hdxp_zMpyBg_w%{>y48AML0yXUleGC=2LY3>qlDc@pmA1+#f~8t3;u63%f&L{2 zDalaQzzk^MZCj0tkS&#Q)dc8jGf>vCG`^cjLkO-E*0Z6yvc5G4a19$k?}Ap;>Yx{m z{ds!}HwuypXv{Zibu4Bso=ysqYDOgF%PsIY~I7 zRg=U^G(o&&-6#v)RtUr}rqI&5d;~H=hAZuQKRjk?Bd#!+eOFzlg}!j+od0uLIbwmEGWf$!9&6s{4%vF zLkl6$-*Rfx`pw*uE9Q+{=_Y%6VLoHTxFxNHwdzU<(92DNh@S(VZB1*}&r?xb&EHDv zc!74ux6)QHjy^AkQOjEdv4L+}$*4-_+Ikr=vMwy`EpeWk-KL*a-<@C@igkDZ%itA1 zfE6paGh#hTV$^fZ@-5u(Y~-?jf7eyKU>?exgq_4eni2CboW7KBo3`4&e((AIU+-YV zgzr@ttKv;butezao=uJL3A`-5AR}9?y(IH#QkM|Ml?=gpBbMR4`bZbSZig{DFfG(; z^zkm@h>_hHsGxU-=q5v<(pwzfuaAzc#G{NYX_y+O55ljjU#Of_UA2$GYFy`GC zQ8JMfxReGmd8FFoyc9#p48{^F0VW=J7fr_dQ3Grqj5`%*RmlwU!B^RAOqR@s^%Niq z{#FK^QTMIG^T4T;7~@jXnLwqx1rH5E_QyxWBq4j=xqq~0;RkZxRe3a498D;r33)tM z7`;&(eOu1It@Qn6p}J}I*az)*+UG9KP23$RRyWDjO)%&i2`($~)4Cu{F7AO$HEW0; zL~YC_vnjxdXY+=ZY>P@usayhK?cg(_CMeJu$iPzOvg0BGVJ7&fb|VY`Xr2=tz5&<^ zeTJ1U78*-pX*K~g^HKfi9Ml)_g(7cB_?|E&R#;A@xb$e8JEQL1k`v`#tPiv!OcL|l zg6*U9M{hlYCYTR3;$oX-c{PKrTHlO$r7#b;BE8VbNK1txt- z>x<>G;@FsK0L-D1Fpd7cC^4fNu%Dq_3M>=Ad!_Vhi6ORiHlVA}kDsCR&%m)iVmPs< z;Mh0WwPtt8?!yIp^JM3$zwMKu#i6CdGJ4<#baDQ^|J(vOm7RWbmYcKPN#A$YEBQ5A$?f=+QJk@{yRR4p4zfewbij$js zZPn(OJiqF!ni-lNnoZ50D>x4pore|Y;f0>1(W0|`id?ljW{yoCo83207VLHNT)}>D zq3x4ni^rCBesXs4>@xQo{xg21=QrYKqI@Y_yfmm>8kFDUikC)}OQT;pylbA_MNhrr zsV{ok6i?gIfu+7rTZ%8guDtyEz508%3NH^8JeQ};t9DPpz6aAgs~&jXSqlET>(j1f z-%9ww(Vl|m9q?v?R2KzT!u1+8MXlPMw@=R1eQf?)KOW+)DY_ZO%@o}Yio41HMkW2w0o&t-gEkif%KmJnjo#Ol25i1u8#lQNjN;S^~#b7# z5n*f1+Aor<1L&MbXx0@5e7QMA4Ek)?%@k-B&<0z@df4iK0c@L>-5#KZD}%z7+Il67 z@$sg#VMGNZu9IvPS<`p%U7Tfi3$Yel$C-2SixrF(t%Nz=H}z5ntcpdz@rRs0&MhA)l>=$qBzWruHo!DUd5`K z;jj{hcorNCg$O1>N$6262?#7^EI|Y0gLrLIng=`UQ? zZ+RQAsT9@h@a;ssm=dEB9#EbLs7gTyl5~*;&u-M%hp-o<8soWiMnV_^BCP(Fy^M?&9I^W&qwvEwm1p-V)3$s=X2%pyVzs_WK6zcK zNv(PJ7QKy%x3TE;DPG^-9V#}RRGLl}o6agtXA9m}i{4(v+gtGVt$Gfv?c6tSRd)J* zUsLA2i*veBakH5Ustm*j9nW+QUV;FBt=(=^%j zcn9IFneSP)$ZuSg--^js<8nX0Rw??wcFG>t5zpLdu!29F5Cw@lVXO z+ceqzwT-a2e{yrp;(RwgbM5EX=4uwI6xZPgmgZH9>-p&8u3ve7z5DNXFLxK(I*V-= zl(q|nwvaqHvf6Y)?&y#>uanAx`-_9Hhh-{?mr;L-ZGxnr^@03%{-ViU>eh1IcBJzA7C4z;2{Dq2 zR6^GrV4>kkq+)$9U{&otM}OYC@dq8v6F(&R@)W~p-kM65Z3o31D%lGfeuOF65S8?4 zZn}@w91HX_*gyI)&hYc`QKdJDpQD7e5zMH?IO#M8I^qtr4B#&OMB#-wYi^HBzP3hN z-(_ca+#a3^$-DfrE%1PDdFZU3BH_BsQnPKd19y(k`sUie8@+k=%3R+9wa~V3W%2Yv z%d&TwTd{n0efjFW1G1~{-z=dg2Eyh2gW0)eaZC-q>j1|By$)-smS`03AR6`UQq9q5 zT#QAdnrcHoh3=B-p+reFND!xRmE!P$au{AyMyi&xXlf2k;}r~G z0W@c@tkWd;g&>Q)I!uO! zTd=%Zv~(+$?t6xUrEk)>>Tu2UPWR6KNO2sTw0`OK%)b0V$DNK3&fGaOA6RNAR39(6 zPfR+%bDL?JZogOfJ)--AUs+Rfi+TCj(3U>wBe`dv5sO>M-4k)z)a%AX1 z?d5_kaliKRTvhRET)7%AULCvLKGimJeERrbw=1=m6EN?r)y56k#7Hwci$gOYQ8e Lr+-`Ph5Ua2ESA06 literal 0 HcmV?d00001 diff --git a/files/alerting-rules.yml b/files/alerting-rules.yml new file mode 100644 index 0000000..0a343f5 --- /dev/null +++ b/files/alerting-rules.yml @@ -0,0 +1,58 @@ +# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml. +# The chart passes alertingRules through Helm's tpl, so every Prometheus template +# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this +# file with lookup('file'), which it does not template. +# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here. +groups: + - name: eom-availability + rules: + - alert: SiteDown + expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500) + for: 5m + labels: + severity: critical + annotations: + summary: '{{`{{ $labels.instance }}`}} is down' + description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).' + - alert: CertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14 + for: 1h + labels: + severity: warning + annotations: + summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days' + description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.' + - alert: ScrapeTargetDown + expr: up == 0 + for: 15m + labels: + severity: warning + annotations: + summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})' + description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.' + + - name: eom-hardware + rules: + - alert: GpuOverheating + expr: DCGM_FI_DEV_GPU_TEMP >= 85 + for: 10m + labels: + severity: critical + annotations: + summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C' + description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.' + - alert: HostOverheating + expr: max by (instance) (node_hwmon_temp_celsius) >= 85 + for: 15m + labels: + severity: warning + annotations: + summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C' + description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.' + - alert: HostRebooted + expr: time() - node_boot_time_seconds < 900 + labels: + severity: info + annotations: + summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago' + description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.' diff --git a/files/forum-digest.py b/files/forum-digest.py new file mode 100644 index 0000000..acfd4b3 --- /dev/null +++ b/files/forum-digest.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Daily digest of unanswered discourse.eom.dev topics, posted to Matrix. + +Lists recent public topics that have no replies yet, skipping pinned, closed +and archived topics, and topics started by staff or EXCLUDE_AUTHORS (by default +Eric and the assistant). Posts one message through grafana-matrix-forwarder (Alertmanager webhook format), and +nothing when every topic has an answer. Standard library only. +""" +import datetime +import hashlib +import json +import os +import sys +import urllib.request + +FORUM = os.environ.get("FORUM_URL", "https://discourse.eom.dev").rstrip("/") +FORWARDER = os.environ["FORWARDER_URL"] +MAX_AGE_DAYS = int(os.environ.get("MAX_AGE_DAYS", "30")) +DRY_RUN = os.environ.get("DRY_RUN") == "1" +# Authors whose topics never need a reply from Eric (his own posts and the assistant's). +EXCLUDE = {u.strip() for u in os.environ.get("EXCLUDE_AUTHORS", "eric,claude").split(",") if u.strip()} + + +def get(path): + req = urllib.request.Request(FORUM + path, headers={"Accept": "application/json", "User-Agent": "eom-forum-digest"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def staff_usernames(): + about = get("/about.json").get("about", {}) + return {u["username"] for key in ("admins", "moderators") for u in about.get(key, [])} + + +def unanswered(): + staff = staff_usernames() + data = get("/latest.json?order=created") + users = {u["id"]: u["username"] for u in data.get("users", [])} + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=MAX_AGE_DAYS) + out = [] + for t in data.get("topic_list", {}).get("topics", []): + if t.get("posts_count", 0) != 1 or t.get("pinned") or t.get("closed") or t.get("archived"): + continue + created = datetime.datetime.fromisoformat(t["created_at"].replace("Z", "+00:00")) + if created < cutoff: + continue + op = next((users.get(p["user_id"]) for p in t.get("posters", []) if "Original Poster" in (p.get("description") or "")), None) + if op in staff or op in EXCLUDE: + continue + age = (datetime.datetime.now(datetime.timezone.utc) - created).days + out.append({"id": t["id"], "title": t["title"], "author": op or "?", "age_days": age, + "url": f"{FORUM}/t/{t['slug']}/{t['id']}"}) + return sorted(out, key=lambda x: -x["age_days"]) + + +def post(topics): + today = datetime.date.today().isoformat() + lines = [f"- {t['title']} by @{t['author']}, {t['age_days']}d old: {t['url']}" for t in topics] + summary = f"{len(topics)} forum topic{'s' if len(topics) != 1 else ''} waiting for a reply:\n" + "\n".join(lines) + alert = { + "status": "firing", + "labels": {"alertname": "ForumUnanswered", "severity": "info", "instance": "discourse.eom.dev"}, + "annotations": {"summary": summary, "description": "Daily forum digest (eom-forum-digest CronJob)."}, + "startsAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": FORUM + "/latest", + # One fingerprint per day: a new message each day rather than a reply thread. + "fingerprint": hashlib.sha1(f"forum-digest-{today}".encode()).hexdigest()[:16], + } + payload = { + "version": "4", "receiver": "matrix", "status": "firing", "groupKey": f"forum-digest-{today}", + "externalURL": FORUM, "groupLabels": {"alertname": "ForumUnanswered"}, + "commonLabels": alert["labels"], "commonAnnotations": alert["annotations"], "alerts": [alert], + } + if DRY_RUN: + print(json.dumps(payload, indent=2)) + return + req = urllib.request.Request(FORWARDER, data=json.dumps(payload).encode(), method="POST", + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + print("forwarder:", r.status) + + +def main(): + topics = unanswered() + print(f"{len(topics)} unanswered topic(s)") + for t in topics: + print(f" {t['id']} {t['title']} ({t['age_days']}d, @{t['author']})") + if topics: + post(topics) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/main.yml b/tasks/main.yml index 63d7ecf..aa261f9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,6 +1,50 @@ #SPDX-License-Identifier: MIT-0 --- # tasks file for ansible-role-prometheus +- name: Deploy blackbox exporter (HTTPS probes of public sites) + kubernetes.core.k8s: + state: present + definition: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: blackbox-exporter + namespace: prometheus + labels: + app: blackbox-exporter + spec: + replicas: 1 + selector: + matchLabels: + app: blackbox-exporter + template: + metadata: + labels: + app: blackbox-exporter + spec: + containers: + - name: blackbox-exporter + image: "prom/blackbox-exporter:{{ prometheus_blackbox_exporter_tag }}" + ports: + - containerPort: 9115 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + - apiVersion: v1 + kind: Service + metadata: + name: blackbox-exporter + namespace: prometheus + spec: + selector: + app: blackbox-exporter + ports: + - port: 9115 + targetPort: 9115 + - name: Deploy Prometheus kubernetes.core.helm: name: prometheus @@ -12,6 +56,18 @@ image: repository: bitnamilegacy/alertmanager tag: 0.28.1-debian-12-r15 + configuration: | + route: + receiver: matrix + group_by: [alertname, instance] + group_wait: 30s + group_interval: 5m + repeat_interval: {{ prometheus_alert_repeat_interval }} + receivers: + - name: matrix + webhook_configs: + - url: "{{ prometheus_matrix_forwarder_url }}" + send_resolved: true server: image: repository: bitnamilegacy/prometheus @@ -26,6 +82,7 @@ memory: 2Gi persistence: size: 256Gi + alertingRules: "{{ lookup('ansible.builtin.file', 'alerting-rules.yml') | from_yaml }}" extraScrapeConfigs: - job_name: local static_configs: @@ -147,3 +204,74 @@ labels: instance: alpha-worker-17 metrics_path: /metrics + + - job_name: blackbox + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: "{{ prometheus_blackbox_targets }}" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter.prometheus.svc.cluster.local:9115 + + - job_name: dcgm + static_configs: + - targets: + - nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400 + +- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix) + kubernetes.core.k8s: + state: present + definition: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: forum-digest + namespace: prometheus + data: + forum-digest.py: "{{ lookup('ansible.builtin.file', 'forum-digest.py') }}" + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: forum-digest + namespace: prometheus + spec: + schedule: "{{ prometheus_forum_digest_schedule }}" + timeZone: America/New_York + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: Never + containers: + - name: forum-digest + image: python:3.13-alpine + command: ["python3", "-I", "/app/forum-digest.py"] + env: + - name: FORWARDER_URL + value: "{{ prometheus_matrix_forwarder_url }}" + - name: EXCLUDE_AUTHORS + value: "{{ prometheus_forum_digest_exclude | join(',') }}" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + volumeMounts: + - name: app + mountPath: /app + volumes: + - name: app + configMap: + name: forum-digest