- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
HostOverheating and HostRebooted. Disk and RAM are already covered by
Grafana rules. Templates use Helm's {{` `}} escaping because the chart
runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
discourse.eom.dev to Matrix through the same forwarder, and nothing when
there are none. Standard-library Python, public Discourse JSON only.
These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
96 lines
4.1 KiB
Python
96 lines
4.1 KiB
Python
#!/usr/bin/env python3
|
|
"""Daily digest of unanswered discourse.eom.dev topics, posted to Matrix.
|
|
|
|
Lists recent public topics that have no replies yet, skipping pinned, closed
|
|
and archived topics, and topics started by staff or EXCLUDE_AUTHORS (by default
|
|
Eric and the assistant). Posts one message through grafana-matrix-forwarder (Alertmanager webhook format), and
|
|
nothing when every topic has an answer. Standard library only.
|
|
"""
|
|
import datetime
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
import urllib.request
|
|
|
|
FORUM = os.environ.get("FORUM_URL", "https://discourse.eom.dev").rstrip("/")
|
|
FORWARDER = os.environ["FORWARDER_URL"]
|
|
MAX_AGE_DAYS = int(os.environ.get("MAX_AGE_DAYS", "30"))
|
|
DRY_RUN = os.environ.get("DRY_RUN") == "1"
|
|
# Authors whose topics never need a reply from Eric (his own posts and the assistant's).
|
|
EXCLUDE = {u.strip() for u in os.environ.get("EXCLUDE_AUTHORS", "eric,claude").split(",") if u.strip()}
|
|
|
|
|
|
def get(path):
|
|
req = urllib.request.Request(FORUM + path, headers={"Accept": "application/json", "User-Agent": "eom-forum-digest"})
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
return json.load(r)
|
|
|
|
|
|
def staff_usernames():
|
|
about = get("/about.json").get("about", {})
|
|
return {u["username"] for key in ("admins", "moderators") for u in about.get(key, [])}
|
|
|
|
|
|
def unanswered():
|
|
staff = staff_usernames()
|
|
data = get("/latest.json?order=created")
|
|
users = {u["id"]: u["username"] for u in data.get("users", [])}
|
|
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=MAX_AGE_DAYS)
|
|
out = []
|
|
for t in data.get("topic_list", {}).get("topics", []):
|
|
if t.get("posts_count", 0) != 1 or t.get("pinned") or t.get("closed") or t.get("archived"):
|
|
continue
|
|
created = datetime.datetime.fromisoformat(t["created_at"].replace("Z", "+00:00"))
|
|
if created < cutoff:
|
|
continue
|
|
op = next((users.get(p["user_id"]) for p in t.get("posters", []) if "Original Poster" in (p.get("description") or "")), None)
|
|
if op in staff or op in EXCLUDE:
|
|
continue
|
|
age = (datetime.datetime.now(datetime.timezone.utc) - created).days
|
|
out.append({"id": t["id"], "title": t["title"], "author": op or "?", "age_days": age,
|
|
"url": f"{FORUM}/t/{t['slug']}/{t['id']}"})
|
|
return sorted(out, key=lambda x: -x["age_days"])
|
|
|
|
|
|
def post(topics):
|
|
today = datetime.date.today().isoformat()
|
|
lines = [f"- {t['title']} by @{t['author']}, {t['age_days']}d old: {t['url']}" for t in topics]
|
|
summary = f"{len(topics)} forum topic{'s' if len(topics) != 1 else ''} waiting for a reply:\n" + "\n".join(lines)
|
|
alert = {
|
|
"status": "firing",
|
|
"labels": {"alertname": "ForumUnanswered", "severity": "info", "instance": "discourse.eom.dev"},
|
|
"annotations": {"summary": summary, "description": "Daily forum digest (eom-forum-digest CronJob)."},
|
|
"startsAt": datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
"endsAt": "0001-01-01T00:00:00Z",
|
|
"generatorURL": FORUM + "/latest",
|
|
# One fingerprint per day: a new message each day rather than a reply thread.
|
|
"fingerprint": hashlib.sha1(f"forum-digest-{today}".encode()).hexdigest()[:16],
|
|
}
|
|
payload = {
|
|
"version": "4", "receiver": "matrix", "status": "firing", "groupKey": f"forum-digest-{today}",
|
|
"externalURL": FORUM, "groupLabels": {"alertname": "ForumUnanswered"},
|
|
"commonLabels": alert["labels"], "commonAnnotations": alert["annotations"], "alerts": [alert],
|
|
}
|
|
if DRY_RUN:
|
|
print(json.dumps(payload, indent=2))
|
|
return
|
|
req = urllib.request.Request(FORWARDER, data=json.dumps(payload).encode(), method="POST",
|
|
headers={"Content-Type": "application/json"})
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
print("forwarder:", r.status)
|
|
|
|
|
|
def main():
|
|
topics = unanswered()
|
|
print(f"{len(topics)} unanswered topic(s)")
|
|
for t in topics:
|
|
print(f" {t['id']} {t['title']} ({t['age_days']}d, @{t['author']})")
|
|
if topics:
|
|
post(topics)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|