diff --git a/defaults/main.yml b/defaults/main.yml index 1564c2b..5523a44 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,3 +1,34 @@ #SPDX-License-Identifier: MIT-0 --- # defaults file for ansible-role-prometheus +prometheus_blackbox_exporter_tag: v0.27.0 +# Grafana-matrix-forwarder endpoint; the room is the existing Grafana alerts room. +prometheus_matrix_forwarder_url: "http://matrix-forwarder.grafana-matrix-forwarder.svc.cluster.local/api/v1/unified?roomId=!HOBNfTFECIjTOhxwDe:eom.dev" +prometheus_alert_repeat_interval: 24h +prometheus_blackbox_targets: + - https://www.eom.dev + - https://eom.dev + - https://discourse.eom.dev + - https://gitea.eom.dev + - https://nextcloud.eom.dev + - https://element.eom.dev + - https://synapse.eom.dev + - https://mas.eom.dev + - https://dex.eom.dev + - https://peertube.eom.dev + - https://owncast.eom.dev + - https://mastodon.eom.dev + - https://listmonk.eom.dev + - https://mailman.eom.dev + - https://grafana.eom.dev + - https://localai.eom.dev + - https://wanderer.eom.dev + - https://wikideck.eom.dev + - https://button.eom.dev + - https://collabora.eom.dev + - https://fediprofile.eom.dev + - https://minio.eom.dev + - https://shkeeper.eom.dev +# Daily digest of unanswered forum topics (9:00 US Eastern). +prometheus_forum_digest_schedule: "0 9 * * *" +prometheus_forum_digest_exclude: [eric, claude] diff --git a/files/__pycache__/forum-digest.cpython-313.pyc b/files/__pycache__/forum-digest.cpython-313.pyc new file mode 100644 index 0000000..1d8a086 Binary files /dev/null and b/files/__pycache__/forum-digest.cpython-313.pyc differ diff --git a/files/alerting-rules.yml b/files/alerting-rules.yml new file mode 100644 index 0000000..0a343f5 --- /dev/null +++ b/files/alerting-rules.yml @@ -0,0 +1,58 @@ +# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml. +# The chart passes alertingRules through Helm's tpl, so every Prometheus template +# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this +# file with lookup('file'), which it does not template. +# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here. +groups: + - name: eom-availability + rules: + - alert: SiteDown + expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500) + for: 5m + labels: + severity: critical + annotations: + summary: '{{`{{ $labels.instance }}`}} is down' + description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).' + - alert: CertificateExpiringSoon + expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14 + for: 1h + labels: + severity: warning + annotations: + summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days' + description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.' + - alert: ScrapeTargetDown + expr: up == 0 + for: 15m + labels: + severity: warning + annotations: + summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})' + description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.' + + - name: eom-hardware + rules: + - alert: GpuOverheating + expr: DCGM_FI_DEV_GPU_TEMP >= 85 + for: 10m + labels: + severity: critical + annotations: + summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C' + description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.' + - alert: HostOverheating + expr: max by (instance) (node_hwmon_temp_celsius) >= 85 + for: 15m + labels: + severity: warning + annotations: + summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C' + description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.' + - alert: HostRebooted + expr: time() - node_boot_time_seconds < 900 + labels: + severity: info + annotations: + summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago' + description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.' diff --git a/files/forum-digest.py b/files/forum-digest.py new file mode 100644 index 0000000..acfd4b3 --- /dev/null +++ b/files/forum-digest.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Daily digest of unanswered discourse.eom.dev topics, posted to Matrix. + +Lists recent public topics that have no replies yet, skipping pinned, closed +and archived topics, and topics started by staff or EXCLUDE_AUTHORS (by default +Eric and the assistant). Posts one message through grafana-matrix-forwarder (Alertmanager webhook format), and +nothing when every topic has an answer. Standard library only. +""" +import datetime +import hashlib +import json +import os +import sys +import urllib.request + +FORUM = os.environ.get("FORUM_URL", "https://discourse.eom.dev").rstrip("/") +FORWARDER = os.environ["FORWARDER_URL"] +MAX_AGE_DAYS = int(os.environ.get("MAX_AGE_DAYS", "30")) +DRY_RUN = os.environ.get("DRY_RUN") == "1" +# Authors whose topics never need a reply from Eric (his own posts and the assistant's). +EXCLUDE = {u.strip() for u in os.environ.get("EXCLUDE_AUTHORS", "eric,claude").split(",") if u.strip()} + + +def get(path): + req = urllib.request.Request(FORUM + path, headers={"Accept": "application/json", "User-Agent": "eom-forum-digest"}) + with urllib.request.urlopen(req, timeout=30) as r: + return json.load(r) + + +def staff_usernames(): + about = get("/about.json").get("about", {}) + return {u["username"] for key in ("admins", "moderators") for u in about.get(key, [])} + + +def unanswered(): + staff = staff_usernames() + data = get("/latest.json?order=created") + users = {u["id"]: u["username"] for u in data.get("users", [])} + cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=MAX_AGE_DAYS) + out = [] + for t in data.get("topic_list", {}).get("topics", []): + if t.get("posts_count", 0) != 1 or t.get("pinned") or t.get("closed") or t.get("archived"): + continue + created = datetime.datetime.fromisoformat(t["created_at"].replace("Z", "+00:00")) + if created < cutoff: + continue + op = next((users.get(p["user_id"]) for p in t.get("posters", []) if "Original Poster" in (p.get("description") or "")), None) + if op in staff or op in EXCLUDE: + continue + age = (datetime.datetime.now(datetime.timezone.utc) - created).days + out.append({"id": t["id"], "title": t["title"], "author": op or "?", "age_days": age, + "url": f"{FORUM}/t/{t['slug']}/{t['id']}"}) + return sorted(out, key=lambda x: -x["age_days"]) + + +def post(topics): + today = datetime.date.today().isoformat() + lines = [f"- {t['title']} by @{t['author']}, {t['age_days']}d old: {t['url']}" for t in topics] + summary = f"{len(topics)} forum topic{'s' if len(topics) != 1 else ''} waiting for a reply:\n" + "\n".join(lines) + alert = { + "status": "firing", + "labels": {"alertname": "ForumUnanswered", "severity": "info", "instance": "discourse.eom.dev"}, + "annotations": {"summary": summary, "description": "Daily forum digest (eom-forum-digest CronJob)."}, + "startsAt": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": FORUM + "/latest", + # One fingerprint per day: a new message each day rather than a reply thread. + "fingerprint": hashlib.sha1(f"forum-digest-{today}".encode()).hexdigest()[:16], + } + payload = { + "version": "4", "receiver": "matrix", "status": "firing", "groupKey": f"forum-digest-{today}", + "externalURL": FORUM, "groupLabels": {"alertname": "ForumUnanswered"}, + "commonLabels": alert["labels"], "commonAnnotations": alert["annotations"], "alerts": [alert], + } + if DRY_RUN: + print(json.dumps(payload, indent=2)) + return + req = urllib.request.Request(FORWARDER, data=json.dumps(payload).encode(), method="POST", + headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(req, timeout=30) as r: + print("forwarder:", r.status) + + +def main(): + topics = unanswered() + print(f"{len(topics)} unanswered topic(s)") + for t in topics: + print(f" {t['id']} {t['title']} ({t['age_days']}d, @{t['author']})") + if topics: + post(topics) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tasks/main.yml b/tasks/main.yml index 63d7ecf..aa261f9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,6 +1,50 @@ #SPDX-License-Identifier: MIT-0 --- # tasks file for ansible-role-prometheus +- name: Deploy blackbox exporter (HTTPS probes of public sites) + kubernetes.core.k8s: + state: present + definition: + - apiVersion: apps/v1 + kind: Deployment + metadata: + name: blackbox-exporter + namespace: prometheus + labels: + app: blackbox-exporter + spec: + replicas: 1 + selector: + matchLabels: + app: blackbox-exporter + template: + metadata: + labels: + app: blackbox-exporter + spec: + containers: + - name: blackbox-exporter + image: "prom/blackbox-exporter:{{ prometheus_blackbox_exporter_tag }}" + ports: + - containerPort: 9115 + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + - apiVersion: v1 + kind: Service + metadata: + name: blackbox-exporter + namespace: prometheus + spec: + selector: + app: blackbox-exporter + ports: + - port: 9115 + targetPort: 9115 + - name: Deploy Prometheus kubernetes.core.helm: name: prometheus @@ -12,6 +56,18 @@ image: repository: bitnamilegacy/alertmanager tag: 0.28.1-debian-12-r15 + configuration: | + route: + receiver: matrix + group_by: [alertname, instance] + group_wait: 30s + group_interval: 5m + repeat_interval: {{ prometheus_alert_repeat_interval }} + receivers: + - name: matrix + webhook_configs: + - url: "{{ prometheus_matrix_forwarder_url }}" + send_resolved: true server: image: repository: bitnamilegacy/prometheus @@ -26,6 +82,7 @@ memory: 2Gi persistence: size: 256Gi + alertingRules: "{{ lookup('ansible.builtin.file', 'alerting-rules.yml') | from_yaml }}" extraScrapeConfigs: - job_name: local static_configs: @@ -147,3 +204,74 @@ labels: instance: alpha-worker-17 metrics_path: /metrics + + - job_name: blackbox + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: "{{ prometheus_blackbox_targets }}" + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter.prometheus.svc.cluster.local:9115 + + - job_name: dcgm + static_configs: + - targets: + - nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400 + +- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix) + kubernetes.core.k8s: + state: present + definition: + - apiVersion: v1 + kind: ConfigMap + metadata: + name: forum-digest + namespace: prometheus + data: + forum-digest.py: "{{ lookup('ansible.builtin.file', 'forum-digest.py') }}" + - apiVersion: batch/v1 + kind: CronJob + metadata: + name: forum-digest + namespace: prometheus + spec: + schedule: "{{ prometheus_forum_digest_schedule }}" + timeZone: America/New_York + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 1 + ttlSecondsAfterFinished: 86400 + template: + spec: + restartPolicy: Never + containers: + - name: forum-digest + image: python:3.13-alpine + command: ["python3", "-I", "/app/forum-digest.py"] + env: + - name: FORWARDER_URL + value: "{{ prometheus_matrix_forwarder_url }}" + - name: EXCLUDE_AUTHORS + value: "{{ prometheus_forum_digest_exclude | join(',') }}" + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + memory: 128Mi + volumeMounts: + - name: app + mountPath: /app + volumes: + - name: app + configMap: + name: forum-digest