SQLite

Dockerfile

@@ -4,6 +4,7 @@ ENV PYTHONUNBUFFERED=1 \
     BUTTON_HOST=0.0.0.0 \
     BUTTON_PORT=8000 \
     BUTTON_DATA_FILE=/data/data.json \
+    BUTTON_DATABASE_FILE=/data/data.sqlite3 \
     BUTTON_MAILING_LIST_FILE=/data/mailing_list.csv
 
 RUN useradd --system --uid 10001 --no-create-home button \

README.md

@@ -35,7 +35,7 @@ Logged-in players can also join the mailing list.
 docker compose up -d --build
 ```
 
-Open http://localhost:8000. Game data and the mailing list are stored in the `button-data` volume.
+Open http://localhost:8000. The SQLite game database and the mailing list are stored in the `button-data` volume. Existing JSON game data in that volume is automatically migrated on the first startup after upgrading.
 
 ```sh
 docker compose logs -f                                    # view logs
@@ -52,7 +52,17 @@ Requires Python 3.9+. No third-party packages.
 python server.py
 ```
 
-Open http://127.0.0.1:8000. Delete `data.json` to start a new game.
+Open http://127.0.0.1:8000. Game state is stored in `data.sqlite3`.
+
+### Migrating existing data
+
+On startup, if the database has no saved game and `data.json` exists, the server automatically imports the countdown, players, password hashes, active sessions, subscription flags, and complete press history in one transaction. Older single-token player records are also supported. No manual conversion or additional dependencies are needed.
+
+The JSON file is left untouched as a backup. Once the database contains a game, it is authoritative: subsequent restarts never re-import the JSON file. Invalid source data causes startup to fail rather than silently starting a new game; fix the source and restart to retry. The mailing-list CSV is unchanged.
+
+Custom `BUTTON_DATA_FILE` paths continue to locate legacy JSON data; by default, the SQLite database is created alongside it with a `.sqlite3` suffix. `BUTTON_DATABASE_FILE` can override the database path. Docker keeps both paths under `/data`, so existing named volumes migrate automatically.
+
+To start a new game, stop the server and delete the SQLite database **and remove or move the legacy JSON backup**; leaving the JSON file in place would import the old game again.
 
 ## Configuration
 
@@ -63,7 +73,8 @@ Set via environment variables (in Docker, add them under `environment:` in `comp
 | `BUTTON_DURATION_SECONDS`   | `86400`                  | Countdown length. Zones scale with it. |
 | `BUTTON_HOST`               | `127.0.0.1` (`0.0.0.0` in Docker) | Address to listen on.       |
 | `BUTTON_PORT`               | `8000`                   | Port to listen on.                   |
-| `BUTTON_DATA_FILE`          | `data.json`              | Game state file.                     |
+| `BUTTON_DATA_FILE`          | `data.json`              | Legacy JSON file to import if no saved SQLite game exists. |
+| `BUTTON_DATABASE_FILE`      | `data.sqlite3`           | SQLite game database; defaults to `BUTTON_DATA_FILE` with a `.sqlite3` suffix. |
 | `BUTTON_MAILING_LIST_FILE`  | `mailing_list.csv`       | Mailing list file.                   |
 
 Tip: `BUTTON_DURATION_SECONDS=60` makes for quick testing.

server.py

@@ -7,8 +7,10 @@ import json
 import os
 import re
 import secrets
+import sqlite3
 import threading
 import time
+from contextlib import contextmanager
 from http import HTTPStatus
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -16,6 +18,7 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parent
 STATIC_DIR = ROOT / "static"
 DATA_FILE = Path(os.environ.get("BUTTON_DATA_FILE", ROOT / "data.json"))
+DATABASE_FILE = Path(os.environ.get("BUTTON_DATABASE_FILE", DATA_FILE.with_suffix(".sqlite3")))
 MAILING_LIST_FILE = Path(os.environ.get("BUTTON_MAILING_LIST_FILE", ROOT / "mailing_list.csv"))
 MAILING_LIST_FIELDS = ["subscribed_at", "player", "first_name", "last_name", "email", "phone"]
 EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$")
@@ -105,26 +108,129 @@ def new_game():
     }
 
 
+@contextmanager
+def database():
+    """Use a separate connection per operation, with atomic writes and FK checks."""
+    connection = sqlite3.connect(DATABASE_FILE)
+    connection.row_factory = sqlite3.Row
+    try:
+        connection.execute("PRAGMA foreign_keys = ON")
+        with connection:
+            yield connection
+    finally:
+        connection.close()
+
+
+def write_state(connection, state):
+    """Persist a complete snapshot inside the caller's transaction."""
+    connection.execute("DELETE FROM presses")
+    connection.execute("DELETE FROM sessions")
+    connection.execute("DELETE FROM players")
+    connection.executemany(
+        "INSERT INTO players (id, name, password_hash, created_at, subscribed) VALUES (?, ?, ?, ?, ?)",
+        [(pid, p["name"], p.get("password_hash"), p["created_at"], p.get("subscribed", False))
+         for pid, p in state["players"].items()],
+    )
+    connection.executemany(
+        "INSERT INTO sessions (player_id, position, token_hash) VALUES (?, ?, ?)",
+        [(pid, position, token_hash) for pid, p in state["players"].items()
+         for position, token_hash in enumerate(p["token_hashes"])],
+    )
+    connection.executemany(
+        "INSERT INTO presses (position, player_id, at, remaining_ms, zone, points) VALUES (?, ?, ?, ?, ?, ?)",
+        [(position, p["player_id"], p["at"], p["remaining_ms"], p["zone"], p["points"])
+         for position, p in enumerate(state["presses"])],
+    )
+    connection.execute(
+        """INSERT INTO game (id, duration_ms, started_at, deadline, ended, ended_at)
+           VALUES (1, ?, ?, ?, ?, ?)
+           ON CONFLICT(id) DO UPDATE SET duration_ms = excluded.duration_ms,
+               started_at = excluded.started_at, deadline = excluded.deadline,
+               ended = excluded.ended, ended_at = excluded.ended_at""",
+        (state["duration_ms"], state["started_at"], state["deadline"], state["ended"], state["ended_at"]),
+    )
+
+
 def load_state():
+    if DATABASE_FILE.resolve() == DATA_FILE.resolve():
+        raise ValueError("BUTTON_DATABASE_FILE must differ from the legacy BUTTON_DATA_FILE")
+    DATABASE_FILE.parent.mkdir(parents=True, exist_ok=True)
+    with database() as connection:
+        connection.executescript("""
+            CREATE TABLE IF NOT EXISTS game (
+                id INTEGER PRIMARY KEY CHECK (id = 1),
+                duration_ms INTEGER NOT NULL,
+                started_at INTEGER NOT NULL,
+                deadline INTEGER NOT NULL,
+                ended INTEGER NOT NULL CHECK (ended IN (0, 1)),
+                ended_at INTEGER
+            );
+            CREATE TABLE IF NOT EXISTS players (
+                id TEXT PRIMARY KEY,
+                name TEXT NOT NULL,
+                password_hash TEXT,
+                created_at INTEGER NOT NULL,
+                subscribed INTEGER NOT NULL CHECK (subscribed IN (0, 1))
+            );
+            CREATE TABLE IF NOT EXISTS sessions (
+                player_id TEXT NOT NULL REFERENCES players(id),
+                position INTEGER NOT NULL,
+                token_hash TEXT NOT NULL UNIQUE,
+                PRIMARY KEY (player_id, position)
+            );
+            CREATE TABLE IF NOT EXISTS presses (
+                position INTEGER PRIMARY KEY,
+                player_id TEXT NOT NULL REFERENCES players(id),
+                at INTEGER NOT NULL,
+                remaining_ms INTEGER NOT NULL,
+                zone TEXT NOT NULL,
+                points INTEGER NOT NULL
+            );
+        """)
+        # Serialize initialization. A committed game row is the migration marker,
+        # not merely the existence of a possibly empty or interrupted database.
+        connection.execute("BEGIN IMMEDIATE")
+        game = connection.execute("SELECT * FROM game WHERE id = 1").fetchone()
+        if game is None:
             if DATA_FILE.exists():
                 with DATA_FILE.open("r", encoding="utf-8") as f:
-            return json.load(f)
+                    loaded = json.load(f)
-    state = new_game()
+                for player in loaded["players"].values():
-    save_state(state)
+                    if "token_hash" in player:  # migrate pre-login data as well
-    return state
+                        player["token_hashes"] = [player.pop("token_hash")]
+            else:
+                loaded = new_game()
+            # Any read/validation/SQL failure aborts startup without consuming the
+            # JSON source or committing a partial game. The next start can retry.
+            write_state(connection, loaded)
+            return loaded
+
+        loaded = dict(game)
+        del loaded["id"]
+        loaded["ended"] = bool(loaded["ended"])
+        loaded["players"] = {}
+        for row in connection.execute("SELECT * FROM players ORDER BY rowid"):
+            player = dict(row)
+            pid = player.pop("id")
+            player["subscribed"] = bool(player["subscribed"])
+            player["token_hashes"] = []
+            loaded["players"][pid] = player
+        for row in connection.execute("SELECT * FROM sessions ORDER BY player_id, position"):
+            loaded["players"][row["player_id"]]["token_hashes"].append(row["token_hash"])
+        loaded["presses"] = []
+        for row in connection.execute("SELECT * FROM presses ORDER BY position"):
+            press = dict(row)
+            del press["position"]
+            loaded["presses"].append(press)
+        return loaded
 
 
 def save_state(state):
-    tmp = DATA_FILE.with_suffix(".tmp")
+    with database() as connection:
-    with tmp.open("w", encoding="utf-8") as f:
+        write_state(connection, state)
-        json.dump(state, f)
-    os.replace(tmp, DATA_FILE)
 
 
 state = load_state()
-for _p in state["players"].values():
-    if "token_hash" in _p:  # migrate pre-login data
-        _p["token_hashes"] = [_p.pop("token_hash")]
 token_index = {th: pid for pid, p in state["players"].items() for th in p["token_hashes"]}
 failed_logins = {}  # lowercase name -> (failure count, first failure ms)
 

test_server.py

@@ -0,0 +1,202 @@
+"""Persistence and migration regression tests; never use the live game files."""
+
+import copy
+import importlib.util
+import json
+import os
+import sqlite3
+import tempfile
+import threading
+import unittest
+from http.client import HTTPConnection
+from http.server import ThreadingHTTPServer
+from pathlib import Path
+from unittest.mock import patch
+
+
+class PersistenceTests(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        with tempfile.TemporaryDirectory() as directory:
+            with patch.dict(os.environ, {
+                "BUTTON_DATA_FILE": str(Path(directory) / "data.json"),
+                "BUTTON_DATABASE_FILE": str(Path(directory) / "data.sqlite3"),
+                "BUTTON_MAILING_LIST_FILE": str(Path(directory) / "mailing.csv"),
+            }):
+                spec = importlib.util.spec_from_file_location("button_test_server", Path(__file__).with_name("server.py"))
+                cls.game = importlib.util.module_from_spec(spec)
+                spec.loader.exec_module(cls.game)
+
+    def setUp(self):
+        self.directory = tempfile.TemporaryDirectory()
+        self.addCleanup(self.directory.cleanup)
+        root = Path(self.directory.name)
+        self.game.DATA_FILE = root / "data.json"
+        self.game.DATABASE_FILE = root / "data.sqlite3"
+        self.game.MAILING_LIST_FILE = root / "mailing.csv"
+        self.game.failed_logins.clear()
+        self.game.subscribed_emails.clear()
+
+    def sample_state(self):
+        state = self.game.new_game()
+        state["players"] = {
+            "alice": {
+                "name": "Alice '); DROP TABLE players; --",
+                "password_hash": "existing-password-hash",
+                "token_hashes": ["first-session", "second-session"],
+                "created_at": 123,
+                "subscribed": True,
+            },
+            "bob": {
+                "name": "Bøb",
+                "password_hash": None,
+                "token_hashes": [],
+                "created_at": 456,
+                "subscribed": False,
+            },
+        }
+        state["presses"] = [
+            {"player_id": "alice", "at": 100, "remaining_ms": 20, "zone": "white", "points": 32},
+            {"player_id": "bob", "at": 100, "remaining_ms": 50, "zone": "black", "points": 16},
+            {"player_id": "alice", "at": 200, "remaining_ms": 90, "zone": "red", "points": 8},
+        ]
+        return state
+
+    def write_legacy(self, state):
+        self.game.DATA_FILE.write_text(json.dumps(state), encoding="utf-8")
+
+    def test_fresh_game_survives_restart_without_creating_json(self):
+        state = self.game.load_state()
+        self.assertEqual(state, self.game.load_state())
+        self.assertTrue(self.game.DATABASE_FILE.exists())
+        self.assertFalse(self.game.DATA_FILE.exists())
+
+    def test_migration_preserves_all_game_data_and_source(self):
+        expected = self.sample_state()
+        expected["ended"] = True
+        expected["ended_at"] = expected["deadline"]
+        self.write_legacy(expected)
+        original = self.game.DATA_FILE.read_bytes()
+        self.assertEqual(expected, self.game.load_state())
+        self.assertEqual(expected, self.game.load_state())
+        self.assertEqual(original, self.game.DATA_FILE.read_bytes())
+        with self.game.database() as connection:
+            self.assertEqual(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok")
+            self.assertEqual(connection.execute("PRAGMA foreign_key_check").fetchall(), [])
+
+    def test_existing_database_ignores_legacy_even_if_invalid(self):
+        self.write_legacy(self.sample_state())
+        current = self.game.load_state()
+        current["deadline"] += 12345
+        current["players"]["alice"]["token_hashes"] = ["new-session"]
+        self.game.save_state(current)
+        self.game.DATA_FILE.write_text("not valid JSON", encoding="utf-8")
+        self.assertEqual(current, self.game.load_state())
+
+    def test_single_token_pre_login_players_migrate(self):
+        source = self.sample_state()
+        alice = source["players"]["alice"]
+        del alice["password_hash"], alice["token_hashes"]
+        alice["token_hash"] = "old-session"
+        del source["players"]["bob"]["subscribed"]
+        self.write_legacy(source)
+        loaded = self.game.load_state()
+        self.assertEqual(loaded["players"]["alice"]["token_hashes"], ["old-session"])
+        reloaded = self.game.load_state()
+        self.assertIsNone(reloaded["players"]["alice"]["password_hash"])
+        self.assertFalse(reloaded["players"]["bob"]["subscribed"])
+        self.assertEqual(reloaded["presses"], source["presses"])
+
+    def test_empty_database_still_imports_json(self):
+        self.game.DATABASE_FILE.touch()
+        expected = self.sample_state()
+        self.write_legacy(expected)
+        self.assertEqual(expected, self.game.load_state())
+
+    def test_invalid_json_fails_without_starting_new_game_and_can_retry(self):
+        self.game.DATA_FILE.write_text("{broken", encoding="utf-8")
+        for _ in range(2):
+            with self.assertRaises(json.JSONDecodeError):
+                self.game.load_state()
+        expected = self.sample_state()
+        self.write_legacy(expected)
+        self.assertEqual(expected, self.game.load_state())
+
+    def test_failed_import_rolls_back_and_can_retry(self):
+        invalid = self.sample_state()
+        invalid["presses"][-1]["player_id"] = "missing-player"
+        self.write_legacy(invalid)
+        with self.assertRaises(sqlite3.IntegrityError):
+            self.game.load_state()
+        with self.game.database() as connection:
+            for table in ("game", "players", "sessions", "presses"):
+                self.assertEqual(connection.execute("SELECT COUNT(*) FROM " + table).fetchone()[0], 0)
+        expected = self.sample_state()
+        self.write_legacy(expected)
+        self.assertEqual(expected, self.game.load_state())
+
+    def test_failed_save_retains_previous_database_snapshot(self):
+        self.write_legacy(self.sample_state())
+        previous = self.game.load_state()
+        invalid = copy.deepcopy(previous)
+        invalid["presses"][0]["player_id"] = "missing-player"
+        with self.assertRaises(sqlite3.IntegrityError):
+            self.game.save_state(invalid)
+        self.assertEqual(previous, self.game.load_state())
+
+    def test_database_and_source_cannot_be_same_file(self):
+        self.write_legacy(self.sample_state())
+        original = self.game.DATA_FILE.read_bytes()
+        self.game.DATABASE_FILE = self.game.DATA_FILE
+        with self.assertRaises(ValueError):
+            self.game.load_state()
+        self.assertEqual(original, self.game.DATA_FILE.read_bytes())
+
+    def test_nested_database_path_is_created(self):
+        self.game.DATABASE_FILE = Path(self.directory.name) / "nested" / "game.sqlite3"
+        initial = self.game.load_state()
+        self.assertEqual(initial, self.game.load_state())
+
+    def test_http_registration_login_press_logout_and_restart(self):
+        self.game.state = self.game.load_state()
+        self.game.token_index = {}
+        httpd = ThreadingHTTPServer(("127.0.0.1", 0), self.game.Handler)
+        thread = threading.Thread(target=httpd.serve_forever, daemon=True)
+        thread.start()
+        self.addCleanup(httpd.server_close)
+        self.addCleanup(thread.join)
+        self.addCleanup(httpd.shutdown)
+
+        def request(path, data, token=None):
+            connection = HTTPConnection(*httpd.server_address, timeout=5)
+            try:
+                headers = {"Content-Type": "application/json"}
+                if token:
+                    headers["Authorization"] = "Bearer " + token
+                connection.request("POST", path, json.dumps(data), headers)
+                response = connection.getresponse()
+                return response.status, json.loads(response.read())
+            finally:
+                connection.close()
+
+        status, registered = request("/api/register", {"name": "Alice", "password": "test-password"})
+        self.assertEqual(status, 201)
+        token = registered["token"]
+        self.assertEqual(request("/api/press", {}, token)[0], 200)
+        self.assertEqual(request("/api/subscribe", {"first_name": "Alice", "email": "alice@example.com"}, token)[0], 200)
+        self.game.state = self.game.load_state()
+        self.game.token_index = {th: pid for pid, p in self.game.state["players"].items() for th in p["token_hashes"]}
+        self.assertEqual(request("/api/press", {}, token)[0], 200)
+        status, logged_in = request("/api/login", {"name": "Alice", "password": "test-password"})
+        self.assertEqual(status, 200)
+        self.assertEqual(request("/api/logout", {}, token)[0], 200)
+        self.assertEqual(request("/api/press", {}, token)[0], 401)
+        loaded = self.game.load_state()
+        player = next(iter(loaded["players"].values()))
+        self.assertTrue(player["subscribed"])
+        self.assertEqual(player["token_hashes"], [self.game.hash_token(logged_in["token"])])
+        self.assertEqual(len(loaded["presses"]), 2)
+
+
+if __name__ == "__main__":
+    unittest.main()