Alerting to Matrix, site probes, GPU metrics and a forum digest

- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
  CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
  HostOverheating and HostRebooted. Disk and RAM are already covered by
  Grafana rules. Templates use Helm's {{` `}} escaping because the chart
  runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
  Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
  plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
  discourse.eom.dev to Matrix through the same forwarder, and nothing when
  there are none. Standard-library Python, public Discourse JSON only.

These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tasks/main.yml

@@ -1,6 +1,50 @@
 #SPDX-License-Identifier: MIT-0
 ---
 # tasks file for ansible-role-prometheus
+- name: Deploy blackbox exporter (HTTPS probes of public sites)
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      - apiVersion: apps/v1
+        kind: Deployment
+        metadata:
+          name: blackbox-exporter
+          namespace: prometheus
+          labels:
+            app: blackbox-exporter
+        spec:
+          replicas: 1
+          selector:
+            matchLabels:
+              app: blackbox-exporter
+          template:
+            metadata:
+              labels:
+                app: blackbox-exporter
+            spec:
+              containers:
+                - name: blackbox-exporter
+                  image: "prom/blackbox-exporter:{{ prometheus_blackbox_exporter_tag }}"
+                  ports:
+                    - containerPort: 9115
+                  resources:
+                    requests:
+                      cpu: 10m
+                      memory: 32Mi
+                    limits:
+                      memory: 128Mi
+      - apiVersion: v1
+        kind: Service
+        metadata:
+          name: blackbox-exporter
+          namespace: prometheus
+        spec:
+          selector:
+            app: blackbox-exporter
+          ports:
+            - port: 9115
+              targetPort: 9115
+
 - name: Deploy Prometheus
   kubernetes.core.helm:
     name: prometheus
@@ -12,6 +56,18 @@
         image: 
           repository: bitnamilegacy/alertmanager
           tag: 0.28.1-debian-12-r15
+        configuration: |
+          route:
+            receiver: matrix
+            group_by: [alertname, instance]
+            group_wait: 30s
+            group_interval: 5m
+            repeat_interval: {{ prometheus_alert_repeat_interval }}
+          receivers:
+            - name: matrix
+              webhook_configs:
+                - url: "{{ prometheus_matrix_forwarder_url }}"
+                  send_resolved: true
       server:
         image: 
           repository: bitnamilegacy/prometheus
@@ -26,6 +82,7 @@
             memory: 2Gi
         persistence:
           size: 256Gi
+        alertingRules: "{{ lookup('ansible.builtin.file', 'alerting-rules.yml') | from_yaml }}"
         extraScrapeConfigs:
           - job_name: local
             static_configs:
@@ -147,3 +204,74 @@
                 labels:
                   instance: alpha-worker-17
             metrics_path: /metrics
+
+          - job_name: blackbox
+            metrics_path: /probe
+            params:
+              module: [http_2xx]
+            static_configs:
+              - targets: "{{ prometheus_blackbox_targets }}"
+            relabel_configs:
+              - source_labels: [__address__]
+                target_label: __param_target
+              - source_labels: [__param_target]
+                target_label: instance
+              - target_label: __address__
+                replacement: blackbox-exporter.prometheus.svc.cluster.local:9115
+
+          - job_name: dcgm
+            static_configs:
+              - targets:
+                - nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400
+
+- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix)
+  kubernetes.core.k8s:
+    state: present
+    definition:
+      - apiVersion: v1
+        kind: ConfigMap
+        metadata:
+          name: forum-digest
+          namespace: prometheus
+        data:
+          forum-digest.py: "{{ lookup('ansible.builtin.file', 'forum-digest.py') }}"
+      - apiVersion: batch/v1
+        kind: CronJob
+        metadata:
+          name: forum-digest
+          namespace: prometheus
+        spec:
+          schedule: "{{ prometheus_forum_digest_schedule }}"
+          timeZone: America/New_York
+          concurrencyPolicy: Forbid
+          successfulJobsHistoryLimit: 3
+          failedJobsHistoryLimit: 3
+          jobTemplate:
+            spec:
+              backoffLimit: 1
+              ttlSecondsAfterFinished: 86400
+              template:
+                spec:
+                  restartPolicy: Never
+                  containers:
+                    - name: forum-digest
+                      image: python:3.13-alpine
+                      command: ["python3", "-I", "/app/forum-digest.py"]
+                      env:
+                        - name: FORWARDER_URL
+                          value: "{{ prometheus_matrix_forwarder_url }}"
+                        - name: EXCLUDE_AUTHORS
+                          value: "{{ prometheus_forum_digest_exclude | join(',') }}"
+                      resources:
+                        requests:
+                          cpu: 10m
+                          memory: 32Mi
+                        limits:
+                          memory: 128Mi
+                      volumeMounts:
+                        - name: app
+                          mountPath: /app
+                  volumes:
+                    - name: app
+                      configMap:
+                        name: forum-digest