Merge pull request 'Publish sanitized P2Pool hashrate at www.eom.dev/p2pool/stats.json' (#1) from p2pool-public-stats into main

Reviewed-on: #1

README.md

@@ -22,6 +22,33 @@ Optional:
   stop earning once they leave its PPLNS window.
 - `monero_p2pool_addpeers` (default `[]`): extra P2Pool peers as `host:port`.
   They must be on the same chain.
+- `monero_p2pool_stats_host` (default `www.eom.dev`): website host that gets
+  the public `/p2pool/stats.json` path. Use `""` to keep stats internal.
+- `monero_p2pool_stats_ingress_class` (default `nginx`), `monero_p2pool_stats_image`
+  (default `python:3.13-alpine`).
+
+Public stats
+------------
+
+P2Pool runs with `--data-api /api --local-api`. Those files include miner IP
+addresses, so they are never served. A `stats` sidecar
+(`files/p2pool_stats.py`, standard library only) copies an allowlist of numeric
+fields into one JSON document:
+
+    {"sidechain": "nano",
+     "node":    {"hashrate_15m", "hashrate_1h", "hashrate_24h", "miners",
+                 "shares_found", "shares_failed", "current_effort",
+                 "average_effort", "reward_share_percent",
+                 "last_share_found_time", "updated_at", "stale"},
+     "pool":    {"hashrate", "miners", "difficulty", "height", "blocks_found",
+                 "last_block_found_time", "pplns_window", "updated_at", "stale"},
+     "network": {"hashrate", "difficulty", "height", "reward_xmr",
+                 "updated_at", "stale"}}
+
+`node` is this P2Pool instance (everyone mining through it); `pool` is the
+whole sidechain. Sections are `null` until P2Pool writes them, and the
+endpoint answers 503 until `node` exists. Run the sidecar's tests with
+`python3 -m unittest discover -s tests`.
 
 Example Playbook
 ----------------

defaults/main.yml

@@ -14,6 +14,15 @@ monero_p2pool_addpeers: []
 
 monerod_rpc_secret_name: monerod-rpc
 
+# Public P2Pool stats: node and sidechain hashrate, connected miner count,
+# shares and network difficulty, served at https://<host>/p2pool/stats.json.
+# Worker IPs and wallet fragments are never included. The path is added to
+# an existing website host so it shares that host's TLS certificate; set the
+# host to "" to keep the stats cluster-internal (service p2pool-stats:8080).
+monero_p2pool_stats_host: www.eom.dev
+monero_p2pool_stats_ingress_class: nginx
+monero_p2pool_stats_image: python:3.13-alpine
+
 # Required (Vault).
 monerod_user: ""
 monerod_password: ""

files/p2pool_stats.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT-0
+"""Publish a sanitized summary of P2Pool's --data-api files over HTTP.
+
+P2Pool writes JSON stats into its --data-api directory. Some of them
+(local/stratum "workers", local/p2p peers) contain miner IP addresses and
+wallet fragments, so the directory itself must never be served. This sidecar
+reads only the files it needs, copies an allowlist of numeric fields, and
+serves the result at a single path. Standard library only.
+"""
+import json
+import os
+import sys
+import threading
+import time
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+MAX_FILE_BYTES = 256 * 1024
+# P2Pool rewrites network/stats on every Monero block (~2 min) and
+# local/stratum as jobs go out; older files are still served but flagged
+# "stale" so the website can say when they were last reported.
+STALE_AFTER = 15 * 60
+MONERO_BLOCK_TIME = 120
+ATOMIC_UNITS = 10 ** 12
+CACHE_SECONDS = 10
+
+
+def _number(value):
+    """Finite, non-negative int/float only; bools and strings are rejected."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    if value != value or value in (float("inf"), float("-inf")) or value < 0:
+        return None
+    return value
+
+
+def read_json(path, now):
+    """Return (data, mtime) or (None, None) if missing, oversized or malformed."""
+    try:
+        with open(path, "rb") as handle:
+            stat = os.fstat(handle.fileno())
+            if stat.st_size > MAX_FILE_BYTES:
+                return None, None
+            data = json.loads(handle.read(MAX_FILE_BYTES + 1))
+    except (OSError, ValueError):
+        return None, None
+    return (data, int(stat.st_mtime)) if isinstance(data, dict) else (None, None)
+
+
+def _pick(source, mapping):
+    return {out: _number(source.get(key)) for out, key in mapping.items()}
+
+
+def _stamp(section, mtime, now):
+    section["updated_at"] = mtime
+    section["stale"] = now - mtime > STALE_AFTER
+    return section
+
+
+def summarize(api_dir, sidechain, now=None):
+    now = time.time() if now is None else now
+    summary = {"version": 1, "generated_at": int(now), "sidechain": sidechain,
+               "node": None, "pool": None, "network": None}
+
+    stratum, mtime = read_json(os.path.join(api_dir, "local", "stratum"), now)
+    if stratum is not None:
+        node = _pick(stratum, {
+            "hashrate_15m": "hashrate_15m", "hashrate_1h": "hashrate_1h",
+            "hashrate_24h": "hashrate_24h", "shares_found": "shares_found",
+            "shares_failed": "shares_failed", "current_effort": "current_effort",
+            "average_effort": "average_effort",
+            "reward_share_percent": "block_reward_share_percent",
+            "last_share_found_time": "last_share_found_time",
+        })
+        # Count workers without copying them: each entry holds an IP address.
+        workers = stratum.get("workers")
+        node["miners"] = len(workers) if isinstance(workers, list) else _number(stratum.get("connections"))
+        summary["node"] = _stamp(node, mtime, now)
+
+    pool, mtime = read_json(os.path.join(api_dir, "pool", "stats"), now)
+    stats = pool.get("pool_statistics") if pool else None
+    if isinstance(stats, dict):
+        summary["pool"] = _pick(stats, {
+            "hashrate": "hashRate", "miners": "miners",
+            "difficulty": "sidechainDifficulty", "height": "sidechainHeight",
+            "blocks_found": "totalBlocksFound",
+            "last_block_found_time": "lastBlockFoundTime",
+            "pplns_window": "pplnsWindowSize",
+        })
+        _stamp(summary["pool"], mtime, now)
+
+    network, mtime = read_json(os.path.join(api_dir, "network", "stats"), now)
+    if network is not None:
+        difficulty = _number(network.get("difficulty"))
+        reward = _number(network.get("reward"))
+        summary["network"] = {
+            "difficulty": difficulty,
+            "hashrate": difficulty / MONERO_BLOCK_TIME if difficulty is not None else None,
+            "height": _number(network.get("height")),
+            "reward_xmr": reward / ATOMIC_UNITS if reward is not None else None,
+        }
+        _stamp(summary["network"], mtime, now)
+    return summary
+
+
+class Cache:
+    def __init__(self, api_dir, sidechain):
+        self.api_dir, self.sidechain = api_dir, sidechain
+        self.lock = threading.Lock()
+        self.body, self.built, self.ok = b"", 0.0, False
+
+    def get(self):
+        with self.lock:
+            now = time.time()
+            if now - self.built >= CACHE_SECONDS:
+                summary = summarize(self.api_dir, self.sidechain, now)
+                self.body = json.dumps(summary, separators=(",", ":")).encode()
+                self.ok = summary["node"] is not None
+                self.built = now
+            return self.body, self.ok
+
+
+def make_handler(cache, path):
+    class Handler(BaseHTTPRequestHandler):
+        server_version = "p2pool-stats"
+        sys_version = ""
+
+        def _send(self, status, body=b"", content_type="text/plain; charset=utf-8"):
+            self.send_response(status)
+            self.send_header("Content-Type", content_type)
+            self.send_header("Content-Length", str(len(body)))
+            self.send_header("Cache-Control", "public, max-age=30")
+            self.send_header("X-Content-Type-Options", "nosniff")
+            self.send_header("Access-Control-Allow-Origin", "*")
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(body)
+
+        def do_GET(self):
+            if self.path.split("?", 1)[0] == "/healthz":
+                return self._send(200, b"ok\n")
+            if self.path.split("?", 1)[0] != path:
+                return self._send(404, b"not found\n")
+            body, ok = cache.get()
+            # 503 tells the website to keep its fallback text, but the body is
+            # still valid JSON so pool/network stats remain inspectable.
+            self._send(200 if ok else 503, body, "application/json")
+
+        do_HEAD = do_GET
+
+        def log_message(self, *args):
+            pass  # Requests carry visitor IPs; nothing here needs them.
+
+    return Handler
+
+
+def main():
+    api_dir = os.environ.get("P2POOL_API_DIR", "/api")
+    sidechain = os.environ.get("P2POOL_CHAIN", "main")
+    path = os.environ.get("STATS_PATH", "/p2pool/stats.json")
+    port = int(os.environ.get("PORT", "8080"))
+    server = ThreadingHTTPServer(("0.0.0.0", port), make_handler(Cache(api_dir, sidechain), path))
+    print(f"serving {path} from {api_dir} on :{port}", file=sys.stderr, flush=True)
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

tasks/main.yml

@@ -198,6 +198,19 @@
                 secret:
                   secretName: monerotls
 
+# Mounted into the stats sidecar; the pod annotation below rolls it on change.
+- name: Store the P2Pool stats sidecar script
+  k8s:
+    state: present
+    definition:
+      apiVersion: v1
+      kind: ConfigMap
+      metadata:
+        name: p2pool-stats
+        namespace: monero
+      data:
+        p2pool_stats.py: "{{ lookup('ansible.builtin.file', 'p2pool_stats.py') }}"
+
 - name: deploy p2pool
   k8s:
     definition:
@@ -215,6 +228,8 @@
           metadata:
             labels:
               app: p2pool
+            annotations:
+              monero/stats-script: "{{ lookup('ansible.builtin.file', 'p2pool_stats.py') | hash('sha256') }}"
           spec:
             containers:
               - name: p2pool
@@ -223,13 +238,16 @@
                 command:
                   - p2pool
                 # P2Pool takes "--opt value" only; "--opt=value" makes it exit.
+                # --data-api/--local-api feed the stats sidecar below.
+                # local/stratum lists worker IPs, so /api is never served.
                 args: >-
                   {{
                     ['--host', 'monerod',
                      '--rpc-port', '18081',
                      '--wallet', monero_p2pool_wallet_address,
                      '--stratum', '0.0.0.0:3333',
-                     '--p2p', '0.0.0.0:37889']
+                     '--p2p', '0.0.0.0:37889',
+                     '--data-api', '/api', '--local-api']
                     + ([] if monero_p2pool_chain == 'main' else ['--' ~ monero_p2pool_chain])
                     + (['--addpeers', monero_p2pool_addpeers | join(',')] if monero_p2pool_addpeers else [])
                     + ['--tls-cert', '/etc/letsencrypt/tls.crt',
@@ -251,6 +269,47 @@
                     mountPath: /home/p2pool
                   - name: tls
                     mountPath: /etc/letsencrypt
+                  - name: api
+                    mountPath: /api
+              # Serves an allowlisted summary of /api at /p2pool/stats.json.
+              - name: stats
+                image: "{{ monero_p2pool_stats_image }}"
+                command: ["python3", "-I", "/app/p2pool_stats.py"]
+                env:
+                  - name: P2POOL_API_DIR
+                    value: /api
+                  - name: P2POOL_CHAIN
+                    value: "{{ monero_p2pool_chain }}"
+                  - name: PYTHONDONTWRITEBYTECODE
+                    value: "1"
+                ports:
+                  - containerPort: 8080
+                    name: stats
+                readinessProbe:
+                  httpGet:
+                    path: /healthz
+                    port: stats
+                resources:
+                  requests:
+                    cpu: 10m
+                    memory: 32Mi
+                  limits:
+                    memory: 64Mi
+                securityContext:
+                  runAsNonRoot: true
+                  runAsUser: 65534
+                  runAsGroup: 65534
+                  readOnlyRootFilesystem: true
+                  allowPrivilegeEscalation: false
+                  capabilities:
+                    drop: [ALL]
+                volumeMounts:
+                  - name: api
+                    mountPath: /api
+                    readOnly: true
+                  - name: stats-app
+                    mountPath: /app
+                    readOnly: true
             volumes:
               - name: p2pool
                 persistentVolumeClaim:
@@ -258,6 +317,12 @@
               - name: tls
                 secret:
                   secretName: tls
+              - name: api
+                emptyDir:
+                  sizeLimit: 64Mi
+              - name: stats-app
+                configMap:
+                  name: p2pool-stats
 
 - name: monero service
   k8s:
@@ -297,3 +362,48 @@
             name: p2poolp2p
         type: LoadBalancer
 
+- name: p2pool stats service
+  k8s:
+    definition:
+      apiVersion: v1
+      kind: Service
+      metadata:
+        name: p2pool-stats
+        namespace: monero
+      spec:
+        selector:
+          app: p2pool
+        ports:
+          - port: 8080
+            targetPort: stats
+            name: http
+        type: ClusterIP
+
+# Adds one exact path to the website's host. ingress-nginx merges it into that
+# host's server block, so it reuses the website's TLS certificate and the
+# homepage can fetch it same-origin. Nothing else on the host is affected.
+- name: Publish P2Pool stats on the website host
+  k8s:
+    state: "{{ 'present' if monero_p2pool_stats_host | length > 0 else 'absent' }}"
+    definition:
+      apiVersion: networking.k8s.io/v1
+      kind: Ingress
+      metadata:
+        name: p2pool-stats
+        namespace: monero
+        annotations:
+          nginx.ingress.kubernetes.io/enable-access-log: "false"
+      spec:
+        ingressClassName: "{{ monero_p2pool_stats_ingress_class }}"
+        rules:
+          - host: "{{ monero_p2pool_stats_host | default('unset', true) }}"
+            http:
+              paths:
+                - path: /p2pool/stats.json
+                  pathType: Exact
+                  backend:
+                    service:
+                      name: p2pool-stats
+                      port:
+                        number: 8080
+

tests/test_p2pool_stats.py

@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: MIT-0
+"""Run with: python3 -m unittest discover -s tests"""
+import http.client
+import json
+import os
+import sys
+import tempfile
+import threading
+import time
+import unittest
+from http.server import ThreadingHTTPServer
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "files"))
+import p2pool_stats  # noqa: E402
+
+STRATUM = {
+    "hashrate_15m": 8123, "hashrate_1h": 7990, "hashrate_24h": 8011,
+    "total_hashes": 123456789, "shares_found": 42, "shares_failed": 1,
+    "average_effort": 97.5, "current_effort": 12.25, "connections": 3,
+    "incoming_connections": 3, "block_reward_share_percent": 0.42,
+    "last_share_found_time": 1794090000,
+    "workers": [
+        "203.0.113.7:51234,4000,123,40000,rig1",
+        "198.51.100.9:40000,3000,456,40000,laptop",
+        "192.0.2.1:3333,1123,789,40000,48mFvxYDSCzM",
+    ],
+}
+POOL = {"pool_list": ["pplns"], "pool_statistics": {
+    "hashRate": 25000000, "miners": 512, "totalHashes": 10 ** 15,
+    "lastBlockFoundTime": 1794000000, "lastBlockFound": 3801000,
+    "totalBlocksFound": 321, "pplnsWeight": 1, "pplnsWindowSize": 2160,
+    "sidechainDifficulty": 160000000, "sidechainHeight": 999999}}
+NETWORK = {"difficulty": 859250000000, "hash": "ab" * 32, "height": 3801526,
+           "reward": 609800000000, "timestamp": 1794098000}
+
+
+def write(root, rel, data, mtime=None):
+    path = os.path.join(root, *rel.split("/"))
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    with open(path, "w") as handle:
+        handle.write(data if isinstance(data, str) else json.dumps(data))
+    if mtime is not None:
+        os.utime(path, (mtime, mtime))
+
+
+class SummarizeTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.root = self.tmp.name
+
+    def tearDown(self):
+        self.tmp.cleanup()
+
+    def full(self):
+        write(self.root, "local/stratum", STRATUM)
+        write(self.root, "pool/stats", POOL)
+        write(self.root, "network/stats", NETWORK)
+
+    def test_full_summary(self):
+        self.full()
+        s = p2pool_stats.summarize(self.root, "nano")
+        self.assertEqual(s["sidechain"], "nano")
+        self.assertEqual(s["node"]["hashrate_15m"], 8123)
+        self.assertEqual(s["node"]["miners"], 3)
+        self.assertEqual(s["node"]["reward_share_percent"], 0.42)
+        self.assertEqual(s["pool"]["hashrate"], 25000000)
+        self.assertEqual(s["pool"]["difficulty"], 160000000)
+        self.assertAlmostEqual(s["network"]["hashrate"], 859250000000 / 120)
+        self.assertAlmostEqual(s["network"]["reward_xmr"], 0.6098)
+
+    def test_never_leaks_worker_addresses_or_wallets(self):
+        self.full()
+        body = json.dumps(p2pool_stats.summarize(self.root, "nano"))
+        for secret in ("203.0.113.7", "198.51.100.9", "192.0.2.1", "rig1", "48mFvx", "workers", "ab" * 32):
+            self.assertNotIn(secret, body)
+
+    def test_missing_files_yield_nulls(self):
+        s = p2pool_stats.summarize(self.root, "mini")
+        self.assertIsNone(s["node"])
+        self.assertIsNone(s["pool"])
+        self.assertIsNone(s["network"])
+
+    def test_malformed_files_are_ignored(self):
+        write(self.root, "local/stratum", "{not json")
+        write(self.root, "pool/stats", {"pool_statistics": "nope"})
+        write(self.root, "network/stats", [1, 2, 3])
+        s = p2pool_stats.summarize(self.root, "nano")
+        self.assertEqual((s["node"], s["pool"], s["network"]), (None, None, None))
+
+    def test_old_files_are_flagged_stale(self):
+        self.full()
+        old = int(time.time() - p2pool_stats.STALE_AFTER - 60)
+        write(self.root, "local/stratum", STRATUM, mtime=old)
+        s = p2pool_stats.summarize(self.root, "nano")
+        self.assertTrue(s["node"]["stale"])
+        self.assertEqual(s["node"]["updated_at"], old)
+        self.assertFalse(s["network"]["stale"])
+
+    def test_oversized_file_is_ignored(self):
+        write(self.root, "local/stratum", " " * (p2pool_stats.MAX_FILE_BYTES + 1))
+        self.assertIsNone(p2pool_stats.summarize(self.root, "nano")["node"])
+
+    def test_non_numeric_fields_become_null(self):
+        bad = dict(STRATUM, hashrate_15m="9999", hashrate_1h=True, hashrate_24h=-5,
+                   shares_found=float("nan"))
+        del bad["workers"]
+        write(self.root, "local/stratum", bad)
+        node = p2pool_stats.summarize(self.root, "nano")["node"]
+        self.assertEqual([node[k] for k in ("hashrate_15m", "hashrate_1h", "hashrate_24h", "shares_found")],
+                         [None, None, None, None])
+        self.assertEqual(node["miners"], 3)  # falls back to connection count
+
+
+class ServerTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        cache = p2pool_stats.Cache(self.tmp.name, "nano")
+        handler = p2pool_stats.make_handler(cache, "/p2pool/stats.json")
+        self.server = ThreadingHTTPServer(("127.0.0.1", 0), handler)
+        threading.Thread(target=self.server.serve_forever, daemon=True).start()
+        self.cache = cache
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.tmp.cleanup()
+
+    def request(self, path, method="GET"):
+        conn = http.client.HTTPConnection("127.0.0.1", self.server.server_address[1], timeout=5)
+        conn.request(method, path)
+        response = conn.getresponse()
+        return response.status, dict(response.getheaders()), response.read()
+
+    def test_unavailable_until_p2pool_writes_stats(self):
+        status, headers, body = self.request("/p2pool/stats.json")
+        self.assertEqual(status, 503)
+        self.assertEqual(headers["Content-Type"], "application/json")
+        self.assertIsNone(json.loads(body)["node"])
+
+    def test_serves_summary_with_safe_headers(self):
+        write(self.tmp.name, "local/stratum", STRATUM)
+        status, headers, body = self.request("/p2pool/stats.json?x=1")
+        self.assertEqual(status, 200)
+        self.assertEqual(headers["X-Content-Type-Options"], "nosniff")
+        self.assertEqual(json.loads(body)["node"]["hashrate_15m"], 8123)
+        status, headers, body = self.request("/p2pool/stats.json", "HEAD")
+        self.assertEqual((status, body), (200, b""))
+
+    def test_only_one_path_is_served(self):
+        write(self.tmp.name, "local/stratum", STRATUM)
+        for path in ("/", "/local/stratum", "/p2pool/../local/stratum", "/p2pool/stats.jsonx"):
+            self.assertEqual(self.request(path)[0], 404, path)
+        self.assertEqual(self.request("/healthz")[0], 200)
+
+
+if __name__ == "__main__":
+    unittest.main()