# Prometheus alerting rules for eom.dev, rendered into the chart's rules.yaml.
# The chart passes alertingRules through Helm's tpl, so every Prometheus template
# is wrapped as {{`{{ ... }}`}} to reach Prometheus unchanged. Ansible reads this
# file with lookup('file'), which it does not template.
# Disk and RAM thresholds are covered by Grafana-managed rules, so they're not repeated here.
groups:
  - name: eom-availability
    rules:
      - alert: SiteDown
        expr: probe_success{job="blackbox"} == 0 and (probe_http_status_code{job="blackbox"} == 0 or probe_http_status_code{job="blackbox"} >= 500)
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: '{{`{{ $labels.instance }}`}} is down'
          description: 'The HTTPS probe of {{`{{ $labels.instance }}`}} has failed with a connection error or a 5xx for 5 minutes. 2xx/3xx/4xx count as up (logins and redirects).'
      - alert: CertificateExpiringSoon
        expr: (probe_ssl_earliest_cert_expiry{job="blackbox"} - time()) / 86400 < 14
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: 'TLS certificate for {{`{{ $labels.instance }}`}} expires in {{`{{ $value | printf "%.0f" }}`}} days'
          description: 'cert-manager should have renewed it 30 days before expiry; check the Certificate and its CertificateRequest.'
      - alert: ScrapeTargetDown
        expr: up == 0
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: 'Prometheus cannot scrape {{`{{ $labels.instance }}`}} (job {{`{{ $labels.job }}`}})'
          description: 'The exporter or the host itself is unreachable. Known on 2026-11-07: node_exporter on poweredge-r350 and alpha-control-plane, and the www and nextcloud exporters.'

  - name: eom-hardware
    rules:
      - alert: GpuOverheating
        expr: DCGM_FI_DEV_GPU_TEMP >= 85
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: '{{`{{ $labels.modelName }}`}} on {{`{{ $labels.Hostname }}`}} is at {{`{{ $value }}`}} °C'
          description: 'The GPU has been at or above 85 °C for 10 minutes and is throttling (LocalAI slows 4–5x). Pause LocalAI agents or reduce load, and fix airflow over the card.'
      - alert: HostOverheating
        expr: max by (instance) (node_hwmon_temp_celsius) >= 85
        for: 15m
        labels:
          severity: warning
        annotations:
          summary: '{{`{{ $labels.instance }}`}} sensor at {{`{{ $value }}`}} °C'
          description: 'A hardware temperature sensor has been at or above 85 °C for 15 minutes.'
      - alert: HostRebooted
        expr: time() - node_boot_time_seconds < 900
        labels:
          severity: info
        annotations:
          summary: '{{`{{ $labels.instance }}`}} rebooted {{`{{ $value | humanizeDuration }}`}} ago'
          description: 'Unexpected? Check the host and power logs. This alert clears 15 minutes after boot.'