Modes: live lobbies and matches survive node restarts

Save live lobbies/drafts/matches to $DATA/game/live.json every 15 s and
synchronously on SIGTERM/SIGINT; restore on start with timers reset. Lobbies
whose mode version or rules changed are dropped. Reported by night-review
QA on #6 (autodeploy restarts killed live matches).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-17 00:21:02 -05:00
co-authored by Claude Opus 5.5
parent c9c901c7f2
commit e2798a7ddc
4 changed files with 144 additions and 1 deletions
+15
View File
@@ -121,6 +121,15 @@ also gets a fresh bearer token per lobby; only its hash is kept. When eom.dev ac
shows peer modes with "Hosted by …" and links to the peer's own site to play there. No node shows peer modes with "Hosted by …" and links to the peer's own site to play there. No node
proxies another node's matches. proxies another node's matches.
### Restarts
Live lobbies, drafts and matches survive a node restart. The node saves them to
`$WIKIDECK_DATA_DIR/game/live.json` every 15 s and on SIGTERM/SIGINT, and restores them on start.
Player tokens keep working, and turn and draft timers restart from the moment the node is back.
A lobby is dropped instead of restored if its mode is no longer hosted with the same `version`
and rules, so a match never continues under different rules. A hard crash can lose up to 15 s
of moves.
## 4. Credit for hosts ## 4. Credit for hosts
Every mode shows **"Hosted by &lt;operator&gt;"** (with the node's origin), its player counts, Every mode shows **"Hosted by &lt;operator&gt;"** (with the node's origin), its player counts,
@@ -255,3 +264,9 @@ be pinned in consensus.
starting with the preview-card pages. A card whose page isn't warm yet gets no bonus. This starting with the preview-card pages. A card whose page isn't warm yet gets no bonus. This
keeps load on the shared, rate-limited Wikipedia egress tiny: a few dozen pages per 6 hours per node. keeps load on the shared, rate-limited Wikipedia egress tiny: a few dozen pages per 6 hours per node.
- Spectating and rematches. - Spectating and rematches.
- **Separate game RPC budget.** `/api/game` sits behind the node's global `/api` admission
budget (`WIKIDECK_RPC_REQUESTS_PER_SECOND`, 10 writes/s node-wide), so a flood of game
traffic can delay chain RPC (and the reverse). At human speed this is fine. The fix is to give
game routes their own `RpcBudget`, which means mounting them ahead of the shared budget in
`server/testnet/http.ts`: a change to that shared file, so it's proposed in #6 rather than made
tonight.
+39
View File
@@ -212,6 +212,45 @@ export class ModeHost {
return r; return r;
} }
/**
* Live lobbies and matches as JSON, so a restart (e.g. a redeploy) doesn't lose games.
* Only the host's own data dir ever holds this; token hashes, never tokens, are stored.
*/
snapshot(): string {
const live = [...this.lobbies.values()].filter(l => l.phase === 'waiting' || l.phase === 'drafting' || l.phase === 'playing');
return JSON.stringify({ schema: 'wikideck-live-lobbies/1', savedAt: this.now(), lobbies: live.map(l => ({
...l, mode: { id: l.mode.id, version: l.mode.version },
players: l.players.map(p => ({ ...p, tokenHash: p.tokenHash.toString('hex') })) })) });
}
/**
* Restore a snapshot written by `snapshot()`. Lobbies whose mode is no longer hosted with the
* same version and rules are dropped. Timers restart from now, so nobody times out because
* the node was down. Returns how many lobbies were restored.
*/
restore(json: string): number {
const data = JSON.parse(json) as { schema?: string; lobbies?: any[] };
if (data.schema !== 'wikideck-live-lobbies/1' || !Array.isArray(data.lobbies)) return 0;
const now = this.now();
let restored = 0;
for (const raw of data.lobbies.slice(0, this.maxLobbies)) {
const h = this.hosted.get(raw?.mode?.id);
if (!h || h.mode.version !== raw.mode.version || JSON.stringify(h.rules) !== JSON.stringify(raw.rules)) continue;
if (typeof raw.id !== 'string' || !LOBBY_ID.test(raw.id) || this.lobbies.has(raw.id)) continue;
if (!['waiting', 'drafting', 'playing'].includes(raw.phase) || !Array.isArray(raw.players) || raw.players.length < 1 || raw.players.length > 2) continue;
const lobby: Lobby = { ...raw, mode: h.mode, rules: h.rules,
players: raw.players.map((p: any) => ({ id: String(p.id), name: String(p.name), seen: now, tokenHash: Buffer.from(String(p.tokenHash), 'hex') })),
lastMove: now, createdAt: raw.phase === 'waiting' ? now : raw.createdAt, turnStartedAt: now };
if (lobby.players.some(p => p.tokenHash.length !== 32)) continue;
if (lobby.draft && lobby.mode.decks.kind === 'draft') lobby.draft.deadline = now + lobby.mode.decks.pickSeconds * 1000;
if (lobby.phase === 'playing' && (lobby.state === undefined || !lobby.decks)) continue;
if (lobby.phase === 'drafting' && !lobby.draft) continue;
this.lobbies.set(lobby.id, lobby);
restored++;
}
return restored;
}
/** Expire idle lobbies, auto-pick stalled drafts, forfeit stalled matches, drop old lobbies. */ /** Expire idle lobbies, auto-pick stalled drafts, forfeit stalled matches, drop old lobbies. */
sweep(): void { sweep(): void {
const now = this.now(); const now = this.now();
+33 -1
View File
@@ -3,6 +3,7 @@
* app.use('/api/game', await (await import('../game/index.ts')).gameRouter({ dataDir, enrichment: wikipediaEnrichment })); * app.use('/api/game', await (await import('../game/index.ts')).gameRouter({ dataDir, enrichment: wikipediaEnrichment }));
*/ */
import { join } from 'node:path'; import { join } from 'node:path';
import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
import type express from 'express'; import type express from 'express';
import { loadModes, hostIdentity, gamePeers } from './config.ts'; import { loadModes, hostIdentity, gamePeers } from './config.ts';
import { ModeDiscovery } from './discovery.ts'; import { ModeDiscovery } from './discovery.ts';
@@ -15,7 +16,7 @@ import { TrendCache } from './trends.ts';
type Enrichment = { get(pageId: number): Promise<{ pageviews: { trendPercent: number | null } }> }; type Enrichment = { get(pageId: number): Promise<{ pageviews: { trendPercent: number | null } }> };
export async function gameRouter(options: { dataDir: string | null; enrichment?: Enrichment; export async function gameRouter(options: { dataDir: string | null; enrichment?: Enrichment;
engine?: EnginePort<any, any, any, any>; env?: NodeJS.ProcessEnv; warmTrends?: boolean }): Promise<express.Router> { engine?: EnginePort<any, any, any, any>; env?: NodeJS.ProcessEnv; warmTrends?: boolean; handleSignals?: boolean }): Promise<express.Router> {
const env = options.env ?? process.env; const env = options.env ?? process.env;
const engine = options.engine ?? (await import('./engine-binding.ts')).engineBinding; const engine = options.engine ?? (await import('./engine-binding.ts')).engineBinding;
const modes = loadModes(env); const modes = loadModes(env);
@@ -39,7 +40,38 @@ export async function gameRouter(options: { dataDir: string | null; enrichment?:
store: new MatchStore(options.dataDir ? join(options.dataDir, 'game') : null), store: new MatchStore(options.dataDir ? join(options.dataDir, 'game') : null),
liveData: trends ? async pageId => trends!.get(pageId) : undefined }); liveData: trends ? async pageId => trends!.get(pageId) : undefined });
const discovery = new ModeDiscovery({ peers: gamePeers(env) }); const discovery = new ModeDiscovery({ peers: gamePeers(env) });
if (options.dataDir) persistLive(host, join(options.dataDir, 'game'), options.handleSignals !== false);
const timer = setInterval(() => host.sweep(), 5_000); const timer = setInterval(() => host.sweep(), 5_000);
timer.unref(); timer.unref();
return createGameRouter(host, discovery); return createGameRouter(host, discovery);
} }
const MAX_LIVE_BYTES = 32 * 1024 * 1024;
/**
* Keep live lobbies and matches across restarts: restore `live.json` on start, save it every
* 15 s and synchronously on SIGTERM/SIGINT. These listeners are registered when the router is
* mounted, before the node's own shutdown handler, so the save finishes before it exits.
*/
export function persistLive(host: ModeHost, directory: string, handleSignals: boolean): { save(): void } {
const file = join(directory, 'live.json');
if (existsSync(file)) {
try {
const text = readFileSync(file, 'utf8');
if (text.length <= MAX_LIVE_BYTES) console.log(`Game modes: restored ${host.restore(text)} live lobbies`);
} catch (e) { console.error(`Game modes: could not restore live lobbies: ${(e as Error).message.slice(0, 200)}`); }
// Keep the file we restored from for inspection; the next save writes a fresh one.
try { renameSync(file, `${file}.restored`); } catch { /* ignore */ }
}
const save = () => {
try {
const tmp = `${file}.tmp`;
writeFileSync(tmp, host.snapshot(), { mode: 0o600 });
renameSync(tmp, file);
} catch (e) { console.error(`Game modes: could not save live lobbies: ${(e as Error).message.slice(0, 200)}`); }
};
const timer = setInterval(save, 15_000);
timer.unref();
if (handleSignals) for (const signal of ['SIGTERM', 'SIGINT'] as const) process.once(signal, save);
return { save };
}
+57
View File
@@ -207,3 +207,60 @@ test('modes api: mode rules are checked beyond the engine schema and rejected, n
assert.doesNotThrow(() => resolveRuleset(DEFAULT_RULESET, { keywordsEnabled: ['viral', 'edition', 'wildcard'] }, parseModeRuleset)); assert.doesNotThrow(() => resolveRuleset(DEFAULT_RULESET, { keywordsEnabled: ['viral', 'edition', 'wildcard'] }, parseModeRuleset));
assert.doesNotThrow(() => resolveRuleset(DEFAULT_RULESET, {}, parseModeRuleset)); assert.doesNotThrow(() => resolveRuleset(DEFAULT_RULESET, {}, parseModeRuleset));
}); });
test('modes api: live matches survive a node restart (snapshot on shutdown, restore on start)', async () => {
const { ModeHost } = await import('../server/game/host.ts');
const { MatchStore } = await import('../server/game/store.ts');
const { engineBinding } = await import('../server/game/engine-binding.ts');
const { persistLive } = await import('../server/game/index.ts');
const { mkdtempSync, rmSync, existsSync } = await import('node:fs');
const { tmpdir } = await import('node:os');
const { join } = await import('node:path');
const dir = mkdtempSync(join(tmpdir(), 'modes-live-'));
try {
const modes = [parseMode({ schema: MODE_SCHEMA_VERSION, id: 'classic', version: 1, name: 'C', description: 'x', decks: { kind: 'starter' } }),
parseMode({ schema: MODE_SCHEMA_VERSION, id: 'draft', version: 1, name: 'D', description: 'x', decks: { kind: 'draft', poolSize: 30 }, rules: { deckSize: 10, openingHand: 4 } })];
const make = () => new ModeHost({ identity: { origin: 'http://x.test', operator: 'X' }, modes, engine: engineBinding, store: new MatchStore(dir) });
const first = make();
persistLive(first, dir, false);
const a = first.createLobby('classic', alice);
const b = await first.join(a.lobbyId, bob);
const play = (first.view(a.lobbyId, a.token) as any).legalActions.find((x: any) => x.type === 'play');
await first.act(a.lobbyId, a.token, play);
const d = first.createLobby('draft', alice);
const e = await first.join(d.lobbyId, bob);
const pick = (first.view(d.lobbyId, d.token) as any).draft;
await first.pick(d.lobbyId, [d.token, e.token][pick.toPick]!, 0);
const waiting = first.createLobby('classic', { name: 'Cy', secret: 'cy-secret-0123456789abcdefghij' });
const before = first.view(a.lobbyId, b.token) as any;
persistLive(first, dir, false).save();
// "Restart": a fresh host restores the snapshot. Same tokens keep working.
const second = make();
persistLive(second, dir, false);
assert.ok(existsSync(join(dir, 'live.json.restored')));
const after = second.view(a.lobbyId, b.token) as any;
assert.equal(after.phase, 'playing');
assert.deepEqual(after.state, before.state, 'match state restored exactly');
assert.equal((second.view(d.lobbyId, d.token) as any).phase, 'drafting');
assert.equal((second.view(waiting.lobbyId) as any).phase, 'waiting');
// Finish the restored match: the record replays.
for (let i = 0; i < 200; i++) {
const v = second.view(a.lobbyId, a.token) as any;
if (v.phase !== 'playing') break;
for (const t of [a.token, b.token]) {
const w = second.view(a.lobbyId, t) as any;
const c = w.legalActions?.find((x: any) => x.type === 'commit');
if (c) await second.act(a.lobbyId, t, c);
}
}
const end = second.view(a.lobbyId, a.token) as any;
assert.equal(end.phase, 'finished');
const rec = second.matchRecord(end.matchId);
assert.equal(replay(rec.rules as any, rec.decks as any, rec.seed, rec.actions.map(x => x.action) as any).result?.winner ?? null, rec.winner);
// Rules changed under a mode → its lobbies are dropped, not run under different rules.
const changed = new ModeHost({ identity: { origin: 'http://x.test', operator: 'X' }, engine: engineBinding, store: new MatchStore(null),
modes: [parseMode({ schema: MODE_SCHEMA_VERSION, id: 'classic', version: 1, name: 'C', description: 'x', decks: { kind: 'starter' }, rules: { turns: 5 } })] });
assert.equal(changed.restore(second.snapshot()), 0);
} finally { rmSync(dir, { recursive: true, force: true }); }
});