diff --git a/Dockerfile b/Dockerfile index 4418ae3..dd7f730 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,6 +4,7 @@ ENV PYTHONUNBUFFERED=1 \ BUTTON_HOST=0.0.0.0 \ BUTTON_PORT=8000 \ BUTTON_DATA_FILE=/data/data.json \ + BUTTON_DATABASE_FILE=/data/data.sqlite3 \ BUTTON_MAILING_LIST_FILE=/data/mailing_list.csv RUN useradd --system --uid 10001 --no-create-home button \ diff --git a/README.md b/README.md index 152191c..367dcdb 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ Logged-in players can also join the mailing list. docker compose up -d --build ``` -Open http://localhost:8000. Game data and the mailing list are stored in the `button-data` volume. +Open http://localhost:8000. The SQLite game database and the mailing list are stored in the `button-data` volume. Existing JSON game data in that volume is automatically migrated on the first startup after upgrading. ```sh docker compose logs -f # view logs @@ -52,7 +52,17 @@ Requires Python 3.9+. No third-party packages. python server.py ``` -Open http://127.0.0.1:8000. Delete `data.json` to start a new game. +Open http://127.0.0.1:8000. Game state is stored in `data.sqlite3`. + +### Migrating existing data + +On startup, if the database has no saved game and `data.json` exists, the server automatically imports the countdown, players, password hashes, active sessions, subscription flags, and complete press history in one transaction. Older single-token player records are also supported. No manual conversion or additional dependencies are needed. + +The JSON file is left untouched as a backup. Once the database contains a game, it is authoritative: subsequent restarts never re-import the JSON file. Invalid source data causes startup to fail rather than silently starting a new game; fix the source and restart to retry. The mailing-list CSV is unchanged. + +Custom `BUTTON_DATA_FILE` paths continue to locate legacy JSON data; by default, the SQLite database is created alongside it with a `.sqlite3` suffix. `BUTTON_DATABASE_FILE` can override the database path. Docker keeps both paths under `/data`, so existing named volumes migrate automatically. + +To start a new game, stop the server and delete the SQLite database **and remove or move the legacy JSON backup**; leaving the JSON file in place would import the old game again. ## Configuration @@ -63,7 +73,8 @@ Set via environment variables (in Docker, add them under `environment:` in `comp | `BUTTON_DURATION_SECONDS` | `86400` | Countdown length. Zones scale with it. | | `BUTTON_HOST` | `127.0.0.1` (`0.0.0.0` in Docker) | Address to listen on. | | `BUTTON_PORT` | `8000` | Port to listen on. | -| `BUTTON_DATA_FILE` | `data.json` | Game state file. | +| `BUTTON_DATA_FILE` | `data.json` | Legacy JSON file to import if no saved SQLite game exists. | +| `BUTTON_DATABASE_FILE` | `data.sqlite3` | SQLite game database; defaults to `BUTTON_DATA_FILE` with a `.sqlite3` suffix. | | `BUTTON_MAILING_LIST_FILE` | `mailing_list.csv` | Mailing list file. | Tip: `BUTTON_DURATION_SECONDS=60` makes for quick testing. diff --git a/server.py b/server.py index cb20146..568e4b0 100644 --- a/server.py +++ b/server.py @@ -7,8 +7,10 @@ import json import os import re import secrets +import sqlite3 import threading import time +from contextlib import contextmanager from http import HTTPStatus from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -16,6 +18,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent STATIC_DIR = ROOT / "static" DATA_FILE = Path(os.environ.get("BUTTON_DATA_FILE", ROOT / "data.json")) +DATABASE_FILE = Path(os.environ.get("BUTTON_DATABASE_FILE", DATA_FILE.with_suffix(".sqlite3"))) MAILING_LIST_FILE = Path(os.environ.get("BUTTON_MAILING_LIST_FILE", ROOT / "mailing_list.csv")) MAILING_LIST_FIELDS = ["subscribed_at", "player", "first_name", "last_name", "email", "phone"] EMAIL_RE = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") @@ -105,26 +108,129 @@ def new_game(): } +@contextmanager +def database(): + """Use a separate connection per operation, with atomic writes and FK checks.""" + connection = sqlite3.connect(DATABASE_FILE) + connection.row_factory = sqlite3.Row + try: + connection.execute("PRAGMA foreign_keys = ON") + with connection: + yield connection + finally: + connection.close() + + +def write_state(connection, state): + """Persist a complete snapshot inside the caller's transaction.""" + connection.execute("DELETE FROM presses") + connection.execute("DELETE FROM sessions") + connection.execute("DELETE FROM players") + connection.executemany( + "INSERT INTO players (id, name, password_hash, created_at, subscribed) VALUES (?, ?, ?, ?, ?)", + [(pid, p["name"], p.get("password_hash"), p["created_at"], p.get("subscribed", False)) + for pid, p in state["players"].items()], + ) + connection.executemany( + "INSERT INTO sessions (player_id, position, token_hash) VALUES (?, ?, ?)", + [(pid, position, token_hash) for pid, p in state["players"].items() + for position, token_hash in enumerate(p["token_hashes"])], + ) + connection.executemany( + "INSERT INTO presses (position, player_id, at, remaining_ms, zone, points) VALUES (?, ?, ?, ?, ?, ?)", + [(position, p["player_id"], p["at"], p["remaining_ms"], p["zone"], p["points"]) + for position, p in enumerate(state["presses"])], + ) + connection.execute( + """INSERT INTO game (id, duration_ms, started_at, deadline, ended, ended_at) + VALUES (1, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET duration_ms = excluded.duration_ms, + started_at = excluded.started_at, deadline = excluded.deadline, + ended = excluded.ended, ended_at = excluded.ended_at""", + (state["duration_ms"], state["started_at"], state["deadline"], state["ended"], state["ended_at"]), + ) + + def load_state(): - if DATA_FILE.exists(): - with DATA_FILE.open("r", encoding="utf-8") as f: - return json.load(f) - state = new_game() - save_state(state) - return state + if DATABASE_FILE.resolve() == DATA_FILE.resolve(): + raise ValueError("BUTTON_DATABASE_FILE must differ from the legacy BUTTON_DATA_FILE") + DATABASE_FILE.parent.mkdir(parents=True, exist_ok=True) + with database() as connection: + connection.executescript(""" + CREATE TABLE IF NOT EXISTS game ( + id INTEGER PRIMARY KEY CHECK (id = 1), + duration_ms INTEGER NOT NULL, + started_at INTEGER NOT NULL, + deadline INTEGER NOT NULL, + ended INTEGER NOT NULL CHECK (ended IN (0, 1)), + ended_at INTEGER + ); + CREATE TABLE IF NOT EXISTS players ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + password_hash TEXT, + created_at INTEGER NOT NULL, + subscribed INTEGER NOT NULL CHECK (subscribed IN (0, 1)) + ); + CREATE TABLE IF NOT EXISTS sessions ( + player_id TEXT NOT NULL REFERENCES players(id), + position INTEGER NOT NULL, + token_hash TEXT NOT NULL UNIQUE, + PRIMARY KEY (player_id, position) + ); + CREATE TABLE IF NOT EXISTS presses ( + position INTEGER PRIMARY KEY, + player_id TEXT NOT NULL REFERENCES players(id), + at INTEGER NOT NULL, + remaining_ms INTEGER NOT NULL, + zone TEXT NOT NULL, + points INTEGER NOT NULL + ); + """) + # Serialize initialization. A committed game row is the migration marker, + # not merely the existence of a possibly empty or interrupted database. + connection.execute("BEGIN IMMEDIATE") + game = connection.execute("SELECT * FROM game WHERE id = 1").fetchone() + if game is None: + if DATA_FILE.exists(): + with DATA_FILE.open("r", encoding="utf-8") as f: + loaded = json.load(f) + for player in loaded["players"].values(): + if "token_hash" in player: # migrate pre-login data as well + player["token_hashes"] = [player.pop("token_hash")] + else: + loaded = new_game() + # Any read/validation/SQL failure aborts startup without consuming the + # JSON source or committing a partial game. The next start can retry. + write_state(connection, loaded) + return loaded + + loaded = dict(game) + del loaded["id"] + loaded["ended"] = bool(loaded["ended"]) + loaded["players"] = {} + for row in connection.execute("SELECT * FROM players ORDER BY rowid"): + player = dict(row) + pid = player.pop("id") + player["subscribed"] = bool(player["subscribed"]) + player["token_hashes"] = [] + loaded["players"][pid] = player + for row in connection.execute("SELECT * FROM sessions ORDER BY player_id, position"): + loaded["players"][row["player_id"]]["token_hashes"].append(row["token_hash"]) + loaded["presses"] = [] + for row in connection.execute("SELECT * FROM presses ORDER BY position"): + press = dict(row) + del press["position"] + loaded["presses"].append(press) + return loaded def save_state(state): - tmp = DATA_FILE.with_suffix(".tmp") - with tmp.open("w", encoding="utf-8") as f: - json.dump(state, f) - os.replace(tmp, DATA_FILE) + with database() as connection: + write_state(connection, state) state = load_state() -for _p in state["players"].values(): - if "token_hash" in _p: # migrate pre-login data - _p["token_hashes"] = [_p.pop("token_hash")] token_index = {th: pid for pid, p in state["players"].items() for th in p["token_hashes"]} failed_logins = {} # lowercase name -> (failure count, first failure ms) diff --git a/test_server.py b/test_server.py new file mode 100755 index 0000000..62eddce --- /dev/null +++ b/test_server.py @@ -0,0 +1,202 @@ +"""Persistence and migration regression tests; never use the live game files.""" + +import copy +import importlib.util +import json +import os +import sqlite3 +import tempfile +import threading +import unittest +from http.client import HTTPConnection +from http.server import ThreadingHTTPServer +from pathlib import Path +from unittest.mock import patch + + +class PersistenceTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + with tempfile.TemporaryDirectory() as directory: + with patch.dict(os.environ, { + "BUTTON_DATA_FILE": str(Path(directory) / "data.json"), + "BUTTON_DATABASE_FILE": str(Path(directory) / "data.sqlite3"), + "BUTTON_MAILING_LIST_FILE": str(Path(directory) / "mailing.csv"), + }): + spec = importlib.util.spec_from_file_location("button_test_server", Path(__file__).with_name("server.py")) + cls.game = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.game) + + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + root = Path(self.directory.name) + self.game.DATA_FILE = root / "data.json" + self.game.DATABASE_FILE = root / "data.sqlite3" + self.game.MAILING_LIST_FILE = root / "mailing.csv" + self.game.failed_logins.clear() + self.game.subscribed_emails.clear() + + def sample_state(self): + state = self.game.new_game() + state["players"] = { + "alice": { + "name": "Alice '); DROP TABLE players; --", + "password_hash": "existing-password-hash", + "token_hashes": ["first-session", "second-session"], + "created_at": 123, + "subscribed": True, + }, + "bob": { + "name": "Bøb", + "password_hash": None, + "token_hashes": [], + "created_at": 456, + "subscribed": False, + }, + } + state["presses"] = [ + {"player_id": "alice", "at": 100, "remaining_ms": 20, "zone": "white", "points": 32}, + {"player_id": "bob", "at": 100, "remaining_ms": 50, "zone": "black", "points": 16}, + {"player_id": "alice", "at": 200, "remaining_ms": 90, "zone": "red", "points": 8}, + ] + return state + + def write_legacy(self, state): + self.game.DATA_FILE.write_text(json.dumps(state), encoding="utf-8") + + def test_fresh_game_survives_restart_without_creating_json(self): + state = self.game.load_state() + self.assertEqual(state, self.game.load_state()) + self.assertTrue(self.game.DATABASE_FILE.exists()) + self.assertFalse(self.game.DATA_FILE.exists()) + + def test_migration_preserves_all_game_data_and_source(self): + expected = self.sample_state() + expected["ended"] = True + expected["ended_at"] = expected["deadline"] + self.write_legacy(expected) + original = self.game.DATA_FILE.read_bytes() + self.assertEqual(expected, self.game.load_state()) + self.assertEqual(expected, self.game.load_state()) + self.assertEqual(original, self.game.DATA_FILE.read_bytes()) + with self.game.database() as connection: + self.assertEqual(connection.execute("PRAGMA integrity_check").fetchone()[0], "ok") + self.assertEqual(connection.execute("PRAGMA foreign_key_check").fetchall(), []) + + def test_existing_database_ignores_legacy_even_if_invalid(self): + self.write_legacy(self.sample_state()) + current = self.game.load_state() + current["deadline"] += 12345 + current["players"]["alice"]["token_hashes"] = ["new-session"] + self.game.save_state(current) + self.game.DATA_FILE.write_text("not valid JSON", encoding="utf-8") + self.assertEqual(current, self.game.load_state()) + + def test_single_token_pre_login_players_migrate(self): + source = self.sample_state() + alice = source["players"]["alice"] + del alice["password_hash"], alice["token_hashes"] + alice["token_hash"] = "old-session" + del source["players"]["bob"]["subscribed"] + self.write_legacy(source) + loaded = self.game.load_state() + self.assertEqual(loaded["players"]["alice"]["token_hashes"], ["old-session"]) + reloaded = self.game.load_state() + self.assertIsNone(reloaded["players"]["alice"]["password_hash"]) + self.assertFalse(reloaded["players"]["bob"]["subscribed"]) + self.assertEqual(reloaded["presses"], source["presses"]) + + def test_empty_database_still_imports_json(self): + self.game.DATABASE_FILE.touch() + expected = self.sample_state() + self.write_legacy(expected) + self.assertEqual(expected, self.game.load_state()) + + def test_invalid_json_fails_without_starting_new_game_and_can_retry(self): + self.game.DATA_FILE.write_text("{broken", encoding="utf-8") + for _ in range(2): + with self.assertRaises(json.JSONDecodeError): + self.game.load_state() + expected = self.sample_state() + self.write_legacy(expected) + self.assertEqual(expected, self.game.load_state()) + + def test_failed_import_rolls_back_and_can_retry(self): + invalid = self.sample_state() + invalid["presses"][-1]["player_id"] = "missing-player" + self.write_legacy(invalid) + with self.assertRaises(sqlite3.IntegrityError): + self.game.load_state() + with self.game.database() as connection: + for table in ("game", "players", "sessions", "presses"): + self.assertEqual(connection.execute("SELECT COUNT(*) FROM " + table).fetchone()[0], 0) + expected = self.sample_state() + self.write_legacy(expected) + self.assertEqual(expected, self.game.load_state()) + + def test_failed_save_retains_previous_database_snapshot(self): + self.write_legacy(self.sample_state()) + previous = self.game.load_state() + invalid = copy.deepcopy(previous) + invalid["presses"][0]["player_id"] = "missing-player" + with self.assertRaises(sqlite3.IntegrityError): + self.game.save_state(invalid) + self.assertEqual(previous, self.game.load_state()) + + def test_database_and_source_cannot_be_same_file(self): + self.write_legacy(self.sample_state()) + original = self.game.DATA_FILE.read_bytes() + self.game.DATABASE_FILE = self.game.DATA_FILE + with self.assertRaises(ValueError): + self.game.load_state() + self.assertEqual(original, self.game.DATA_FILE.read_bytes()) + + def test_nested_database_path_is_created(self): + self.game.DATABASE_FILE = Path(self.directory.name) / "nested" / "game.sqlite3" + initial = self.game.load_state() + self.assertEqual(initial, self.game.load_state()) + + def test_http_registration_login_press_logout_and_restart(self): + self.game.state = self.game.load_state() + self.game.token_index = {} + httpd = ThreadingHTTPServer(("127.0.0.1", 0), self.game.Handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + self.addCleanup(httpd.server_close) + self.addCleanup(thread.join) + self.addCleanup(httpd.shutdown) + + def request(path, data, token=None): + connection = HTTPConnection(*httpd.server_address, timeout=5) + try: + headers = {"Content-Type": "application/json"} + if token: + headers["Authorization"] = "Bearer " + token + connection.request("POST", path, json.dumps(data), headers) + response = connection.getresponse() + return response.status, json.loads(response.read()) + finally: + connection.close() + + status, registered = request("/api/register", {"name": "Alice", "password": "test-password"}) + self.assertEqual(status, 201) + token = registered["token"] + self.assertEqual(request("/api/press", {}, token)[0], 200) + self.assertEqual(request("/api/subscribe", {"first_name": "Alice", "email": "alice@example.com"}, token)[0], 200) + self.game.state = self.game.load_state() + self.game.token_index = {th: pid for pid, p in self.game.state["players"].items() for th in p["token_hashes"]} + self.assertEqual(request("/api/press", {}, token)[0], 200) + status, logged_in = request("/api/login", {"name": "Alice", "password": "test-password"}) + self.assertEqual(status, 200) + self.assertEqual(request("/api/logout", {}, token)[0], 200) + self.assertEqual(request("/api/press", {}, token)[0], 401) + loaded = self.game.load_state() + player = next(iter(loaded["players"].values())) + self.assertTrue(player["subscribed"]) + self.assertEqual(player["token_hashes"], [self.game.hash_token(logged_in["token"])]) + self.assertEqual(len(loaded["presses"]), 2) + + +if __name__ == "__main__": + unittest.main()