claudeandClaude Opus 5.5
997b18e7c9
- Prometheus alert rules (files/alerting-rules.yml): SiteDown,
CertificateExpiringSoon, ScrapeTargetDown, GpuOverheating (>= 85 C),
HostOverheating and HostRebooted. Disk and RAM are already covered by
Grafana rules. Templates use Helm's {{` `}} escaping because the chart
runs alertingRules through tpl.
- Alertmanager routes everything to the grafana-matrix-forwarder (same
Matrix room as Grafana alerts), resolved notifications on, repeat 24h.
- Blackbox exporter (prom/blackbox-exporter) probing every public site,
plus a scrape job for the NVIDIA DCGM exporter (T4 temperature, clocks).
- forum-digest CronJob (9:00 ET): posts unanswered member topics on
discourse.eom.dev to Matrix through the same forwarder, and nothing when
there are none. Standard-library Python, public Discourse JSON only.
These replace the LocalAI SRE and community agents, which were too slow
and unreliable on the throttled T4.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
278 lines
8.7 KiB
YAML
278 lines
8.7 KiB
YAML
#SPDX-License-Identifier: MIT-0
|
|
---
|
|
# tasks file for ansible-role-prometheus
|
|
- name: Deploy blackbox exporter (HTTPS probes of public sites)
|
|
kubernetes.core.k8s:
|
|
state: present
|
|
definition:
|
|
- apiVersion: apps/v1
|
|
kind: Deployment
|
|
metadata:
|
|
name: blackbox-exporter
|
|
namespace: prometheus
|
|
labels:
|
|
app: blackbox-exporter
|
|
spec:
|
|
replicas: 1
|
|
selector:
|
|
matchLabels:
|
|
app: blackbox-exporter
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: blackbox-exporter
|
|
spec:
|
|
containers:
|
|
- name: blackbox-exporter
|
|
image: "prom/blackbox-exporter:{{ prometheus_blackbox_exporter_tag }}"
|
|
ports:
|
|
- containerPort: 9115
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 128Mi
|
|
- apiVersion: v1
|
|
kind: Service
|
|
metadata:
|
|
name: blackbox-exporter
|
|
namespace: prometheus
|
|
spec:
|
|
selector:
|
|
app: blackbox-exporter
|
|
ports:
|
|
- port: 9115
|
|
targetPort: 9115
|
|
|
|
- name: Deploy Prometheus
|
|
kubernetes.core.helm:
|
|
name: prometheus
|
|
chart_ref: bitnami/prometheus
|
|
release_namespace: prometheus
|
|
create_namespace: true
|
|
values:
|
|
alertmanager:
|
|
image:
|
|
repository: bitnamilegacy/alertmanager
|
|
tag: 0.28.1-debian-12-r15
|
|
configuration: |
|
|
route:
|
|
receiver: matrix
|
|
group_by: [alertname, instance]
|
|
group_wait: 30s
|
|
group_interval: 5m
|
|
repeat_interval: {{ prometheus_alert_repeat_interval }}
|
|
receivers:
|
|
- name: matrix
|
|
webhook_configs:
|
|
- url: "{{ prometheus_matrix_forwarder_url }}"
|
|
send_resolved: true
|
|
server:
|
|
image:
|
|
repository: bitnamilegacy/prometheus
|
|
tag: 3.5.0-debian-12-r3
|
|
resourcesPreset: none
|
|
resources:
|
|
requests:
|
|
cpu: 250m
|
|
memory: 1Gi
|
|
limits:
|
|
cpu: 1
|
|
memory: 2Gi
|
|
persistence:
|
|
size: 256Gi
|
|
alertingRules: "{{ lookup('ansible.builtin.file', 'alerting-rules.yml') | from_yaml }}"
|
|
extraScrapeConfigs:
|
|
- job_name: local
|
|
static_configs:
|
|
- targets:
|
|
- nextcloud-metrics.nextcloud.svc.cluster.local:9205
|
|
labels:
|
|
instance: nextcloud
|
|
- targets:
|
|
- ingress-nginx-controller-metrics.ingress-nginx.svc.cluster.local:10254
|
|
|
|
- job_name: www
|
|
static_configs:
|
|
- targets:
|
|
- prometheus.www.svc.cluster.local:9117
|
|
labels:
|
|
instance: www
|
|
metrics_path: /metrics
|
|
|
|
- job_name: libvirt_exporter
|
|
static_configs:
|
|
- targets:
|
|
- poweredge-t640:9177
|
|
labels:
|
|
instance: poweredge-t640
|
|
- targets:
|
|
- poweredge-r720:9177
|
|
labels:
|
|
instance: poweredge-r720
|
|
metrics_path: /metrics
|
|
|
|
- job_name: node_exporter
|
|
static_configs:
|
|
- targets:
|
|
- poweredge-r350:9100
|
|
labels:
|
|
instance: poweredge-r350
|
|
- targets:
|
|
- poweredge-t640:9100
|
|
labels:
|
|
instance: poweredge-t640
|
|
- targets:
|
|
- poweredge-r720:9100
|
|
labels:
|
|
instance: poweredge-r720
|
|
- targets:
|
|
- alpha-control-plane:9100
|
|
labels:
|
|
instance: alpha-control-plane
|
|
- targets:
|
|
- alpha-worker-0:9100
|
|
labels:
|
|
instance: alpha-worker-0
|
|
- targets:
|
|
- alpha-worker-1:9100
|
|
labels:
|
|
instance: alpha-worker-1
|
|
- targets:
|
|
- alpha-worker-2:9100
|
|
labels:
|
|
instance: alpha-worker-2
|
|
- targets:
|
|
- alpha-worker-3:9100
|
|
labels:
|
|
instance: alpha-worker-3
|
|
- targets:
|
|
- alpha-worker-4:9100
|
|
labels:
|
|
instance: alpha-worker-4
|
|
- targets:
|
|
- alpha-worker-5:9100
|
|
labels:
|
|
instance: alpha-worker-5
|
|
- targets:
|
|
- alpha-worker-6:9100
|
|
labels:
|
|
instance: alpha-worker-6
|
|
- targets:
|
|
- alpha-worker-7:9100
|
|
labels:
|
|
instance: alpha-worker-7
|
|
- targets:
|
|
- alpha-worker-8:9100
|
|
labels:
|
|
instance: alpha-worker-8
|
|
- targets:
|
|
- alpha-worker-9:9100
|
|
labels:
|
|
instance: alpha-worker-9
|
|
- targets:
|
|
- alpha-worker-10:9100
|
|
labels:
|
|
instance: alpha-worker-10
|
|
- targets:
|
|
- alpha-worker-11:9100
|
|
labels:
|
|
instance: alpha-worker-11
|
|
- targets:
|
|
- alpha-worker-12:9100
|
|
labels:
|
|
instance: alpha-worker-12
|
|
- targets:
|
|
- alpha-worker-13:9100
|
|
labels:
|
|
instance: alpha-worker-13
|
|
- targets:
|
|
- alpha-worker-14:9100
|
|
labels:
|
|
instance: alpha-worker-14
|
|
- targets:
|
|
- alpha-worker-15:9100
|
|
labels:
|
|
instance: alpha-worker-15
|
|
- targets:
|
|
- alpha-worker-16:9100
|
|
labels:
|
|
instance: alpha-worker-16
|
|
- targets:
|
|
- alpha-worker-17:9100
|
|
labels:
|
|
instance: alpha-worker-17
|
|
metrics_path: /metrics
|
|
|
|
- job_name: blackbox
|
|
metrics_path: /probe
|
|
params:
|
|
module: [http_2xx]
|
|
static_configs:
|
|
- targets: "{{ prometheus_blackbox_targets }}"
|
|
relabel_configs:
|
|
- source_labels: [__address__]
|
|
target_label: __param_target
|
|
- source_labels: [__param_target]
|
|
target_label: instance
|
|
- target_label: __address__
|
|
replacement: blackbox-exporter.prometheus.svc.cluster.local:9115
|
|
|
|
- job_name: dcgm
|
|
static_configs:
|
|
- targets:
|
|
- nvidia-dcgm-exporter.gpu-operator.svc.cluster.local:9400
|
|
|
|
- name: Deploy the daily forum digest (unanswered Discourse topics to Matrix)
|
|
kubernetes.core.k8s:
|
|
state: present
|
|
definition:
|
|
- apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: forum-digest
|
|
namespace: prometheus
|
|
data:
|
|
forum-digest.py: "{{ lookup('ansible.builtin.file', 'forum-digest.py') }}"
|
|
- apiVersion: batch/v1
|
|
kind: CronJob
|
|
metadata:
|
|
name: forum-digest
|
|
namespace: prometheus
|
|
spec:
|
|
schedule: "{{ prometheus_forum_digest_schedule }}"
|
|
timeZone: America/New_York
|
|
concurrencyPolicy: Forbid
|
|
successfulJobsHistoryLimit: 3
|
|
failedJobsHistoryLimit: 3
|
|
jobTemplate:
|
|
spec:
|
|
backoffLimit: 1
|
|
ttlSecondsAfterFinished: 86400
|
|
template:
|
|
spec:
|
|
restartPolicy: Never
|
|
containers:
|
|
- name: forum-digest
|
|
image: python:3.13-alpine
|
|
command: ["python3", "-I", "/app/forum-digest.py"]
|
|
env:
|
|
- name: FORWARDER_URL
|
|
value: "{{ prometheus_matrix_forwarder_url }}"
|
|
- name: EXCLUDE_AUTHORS
|
|
value: "{{ prometheus_forum_digest_exclude | join(',') }}"
|
|
resources:
|
|
requests:
|
|
cpu: 10m
|
|
memory: 32Mi
|
|
limits:
|
|
memory: 128Mi
|
|
volumeMounts:
|
|
- name: app
|
|
mountPath: /app
|
|
volumes:
|
|
- name: app
|
|
configMap:
|
|
name: forum-digest
|