Files
wikideck/scripts/sim/stats.ts
T
claudeandClaude Opus 5.5 9841622ac7 sim-balance: bots (random/greedy/lookahead), sim runner and stats
Bots are pure (state, player, rng) => action over an EngineAdapter;
shared/bots/game-adapter.ts adapts the shared/game API (hidden staged
plays, determinized lookahead). scripts/sim/run.ts runs N matches per
ruleset across child processes and writes per-ruleset markdown/JSON.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 22:54:53 -05:00

198 lines
10 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Aggregate MatchRecords into balance statistics and a markdown section.
import type { EngineAdapter, MatchRecord, Seat } from '../../shared/bots/types.ts';
export interface Rate { n: number; wins: number; rate: number; lo: number; hi: number }
/** Wilson 95% interval. */
export function rate(wins: number, n: number): Rate {
if (!n) return { n, wins, rate: NaN, lo: NaN, hi: NaN };
const z = 1.96, p = wins / n, d = 1 + z * z / n;
const c = (p + z * z / (2 * n)) / d, h = (z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))) / d;
return { n, wins, rate: p, lo: c - h, hi: c + h };
}
const quant = (xs: number[], q: number) => {
if (!xs.length) return NaN;
const s = [...xs].sort((a, b) => a - b);
return s[Math.min(s.length - 1, Math.floor(q * s.length))];
};
const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / (xs.length || 1);
function pearson(xs: number[], ys: number[]): number {
const mx = mean(xs), my = mean(ys);
let sxy = 0, sxx = 0, syy = 0;
for (let i = 0; i < xs.length; i++) { const dx = xs[i] - mx, dy = ys[i] - my; sxy += dx * dy; sxx += dx * dx; syy += dy * dy; }
return sxx && syy ? sxy / Math.sqrt(sxx * syy) : NaN;
}
export interface CardStat { id: string; label?: string; games: number; plays: number; wins: number; wr: Rate; delta: number }
export interface RulesetStats {
ruleset: string;
matches: number;
draws: number;
truncated: number;
errors: number;
turns: { mean: number; p10: number; p50: number; p90: number };
firstPlayer: Rate;
early: { decidedMedianFrac: number; decidedByHalf: number; decidedByThird: number };
byBotPair: Record<string, { n: number; seat0: Rate; firstPlayer: Rate; turns: number }>;
skill: Record<string, Rate>; // "lookahead>greedy": win rate of the stronger bot
byDeck: Record<string, Rate>; // excluding mirrors
byTag: Record<string, Rate>; // excluding games where both sides share the tag
matchups: Record<string, Record<string, Rate>>;
cards: CardStat[];
statCorrelation: Record<string, { r: number; n: number }>;
reasons: Record<string, number>;
}
/** Only bot-symmetric games are used for deck/tag/card/first-player balance. */
const symmetric = (m: MatchRecord) => m.bots[0] === m.bots[1];
export function aggregate(records: MatchRecord[], errors = 0, engine?: EngineAdapter<any, any>): RulesetStats {
const ruleset = records[0]?.ruleset ?? '?';
const decisive = records.filter(m => m.winner !== null);
const sym = decisive.filter(symmetric);
const reasons: Record<string, number> = {};
for (const m of records) reasons[m.reason ?? 'none'] = (reasons[m.reason ?? 'none'] ?? 0) + 1;
const turns = records.map(m => m.turns);
const fracs = decisive.filter(m => m.decidedTurn !== null && m.turns > 1).map(m => (m.decidedTurn! - 1) / Math.max(1, m.turns - 1));
const byBotPair: RulesetStats['byBotPair'] = {};
const pairs = new Map<string, MatchRecord[]>();
for (const m of records) { const k = `${m.bots[0]} vs ${m.bots[1]}`; pairs.set(k, [...(pairs.get(k) ?? []), m]); }
for (const [k, ms] of pairs) {
const d = ms.filter(m => m.winner !== null);
byBotPair[k] = {
n: ms.length,
seat0: rate(d.filter(m => m.winner === 0).length, d.length),
firstPlayer: rate(d.filter(m => m.winner === m.firstSeat).length, d.length),
turns: mean(ms.map(m => m.turns)),
};
}
const skill: Record<string, Rate> = {};
const order = ['random', 'greedy', 'lookahead-lite', 'lookahead'];
for (const m of decisive) {
if (symmetric(m)) continue;
const [a, b] = m.bots;
const strong: Seat = order.indexOf(a) >= order.indexOf(b) ? 0 : 1;
const k = `${m.bots[strong]}>${m.bots[1 - strong]}`;
const r = skill[k] ?? (skill[k] = rate(0, 0));
r.n++; if (m.winner === strong) r.wins++;
}
for (const k of Object.keys(skill)) skill[k] = rate(skill[k].wins, skill[k].n);
const tally = (map: Record<string, { w: number; n: number }>, key: string, won: boolean) => {
const t = map[key] ?? (map[key] = { w: 0, n: 0 }); t.n++; if (won) t.w++;
};
const deckT: Record<string, { w: number; n: number }> = {}, tagT: Record<string, { w: number; n: number }> = {};
const mu: Record<string, Record<string, { w: number; n: number }>> = {};
const cardT = new Map<string, { games: number; plays: number; wins: number }>();
for (const m of sym) {
for (const seat of [0, 1] as Seat[]) {
const won = m.winner === seat, o = (1 - seat) as Seat;
if (m.decks[0] !== m.decks[1]) tally(deckT, m.decks[seat], won);
for (const t of m.deckTags[seat]) if (!m.deckTags[o].includes(t)) tally(tagT, t, won);
tally(mu[m.decks[seat]] ?? (mu[m.decks[seat]] = {}), m.decks[o], won);
const seen = new Set<string>();
for (const c of m.cardsPlayed[seat]) {
const cs = cardT.get(c) ?? { games: 0, plays: 0, wins: 0 };
cs.plays++;
if (!seen.has(c)) { seen.add(c); cs.games++; if (won) cs.wins++; }
cardT.set(c, cs);
}
}
}
const toRates = (t: Record<string, { w: number; n: number }>) => Object.fromEntries(Object.entries(t).map(([k, v]) => [k, rate(v.w, v.n)]));
const cards: CardStat[] = [...cardT].map(([id, c]) => {
const wr = rate(c.wins, c.games);
const label = engine?.cardMeta?.(id)?.label;
return { id, label: typeof label === 'string' ? label : undefined, ...c, wr, delta: wr.rate - 0.5 };
}).sort((a, b) => b.delta - a.delta);
const statCorrelation: RulesetStats['statCorrelation'] = {};
if (engine?.cardMeta) {
const eligible = cards.filter(c => c.games >= 30);
const keys = new Set<string>();
const metas = eligible.map(c => engine.cardMeta!(c.id) ?? {});
for (const m of metas) for (const [k, v] of Object.entries(m)) if (typeof v === 'number') keys.add(k);
for (const k of keys) {
const xs: number[] = [], ys: number[] = [];
eligible.forEach((c, i) => { const v = metas[i][k]; if (typeof v === 'number') { xs.push(v); ys.push(c.wr.rate); } });
statCorrelation[k] = { r: pearson(xs, ys), n: xs.length };
}
// keyword / faction tags on cards
const tagW: Record<string, { w: number; n: number }> = {};
eligible.forEach((c, i) => {
for (const [k, v] of Object.entries(metas[i])) {
const vals = Array.isArray(v) ? v : typeof v === 'string' ? [v] : [];
if (k === 'label') continue;
for (const x of vals) { const t = tagW[`${k}:${x}`] ?? (tagW[`${k}:${x}`] = { w: 0, n: 0 }); t.w += c.wins; t.n += c.games; }
}
});
for (const [k, v] of Object.entries(tagW)) statCorrelation[`card ${k}`] = { r: v.w / v.n - 0.5, n: v.n };
}
return {
ruleset, matches: records.length, errors,
draws: records.filter(m => m.winner === null).length,
truncated: records.filter(m => m.truncated).length,
turns: { mean: mean(turns), p10: quant(turns, 0.1), p50: quant(turns, 0.5), p90: quant(turns, 0.9) },
firstPlayer: rate(sym.filter(m => m.winner === m.firstSeat).length, sym.length),
early: {
decidedMedianFrac: quant(fracs, 0.5),
decidedByHalf: fracs.filter(f => f <= 0.5).length / (fracs.length || 1),
decidedByThird: fracs.filter(f => f <= 1 / 3).length / (fracs.length || 1),
},
byBotPair, skill, byDeck: toRates(deckT), byTag: toRates(tagT),
matchups: Object.fromEntries(Object.entries(mu).map(([k, v]) => [k, toRates(v)])),
cards, statCorrelation, reasons,
};
}
const pct = (x: number) => (Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : 'n/a');
const fmtRate = (r: Rate) => `${pct(r.rate)} [${pct(r.lo)}–${pct(r.hi)}] (n=${r.n})`;
const sig = (r: Rate) => (r.lo > 0.5 ? ' **↑**' : r.hi < 0.5 ? ' **↓**' : '');
export function toMarkdown(s: RulesetStats): string {
const L: string[] = [];
L.push(`### Ruleset \`${s.ruleset}\``, '');
L.push(`- Matches: **${s.matches}** (draws ${s.draws}, hit action cap ${s.truncated}, engine errors ${s.errors})`);
L.push(`- First-player win rate (bot-symmetric games): **${fmtRate(s.firstPlayer)}**${sig(s.firstPlayer)}`);
L.push(`- Match length (turns, both players): mean ${s.turns.mean.toFixed(1)}, p10 ${s.turns.p10}, median ${s.turns.p50}, p90 ${s.turns.p90}`);
L.push(`- Decided early: median game is locked in at ${pct(s.early.decidedMedianFrac)} of its length; ${pct(s.early.decidedByThird)} decided in the first third, ${pct(s.early.decidedByHalf)} by half-time`);
L.push(`- End reasons: ${Object.entries(s.reasons).map(([k, v]) => `${k} ${v}`).join(', ')}`, '');
L.push('| Bot pairing | n | seat-0 WR | first-player WR | mean turns |', '|---|---|---|---|---|');
for (const [k, v] of Object.entries(s.byBotPair)) L.push(`| ${k} | ${v.n} | ${fmtRate(v.seat0)} | ${fmtRate(v.firstPlayer)} | ${v.turns.toFixed(1)} |`);
if (Object.keys(s.skill).length) {
L.push('', '**Skill expression** (stronger bot win rate):', '');
for (const [k, v] of Object.entries(s.skill)) L.push(`- ${k}: ${fmtRate(v)}`);
}
L.push('', '| Deck | WR vs other decks |', '|---|---|');
for (const [k, v] of Object.entries(s.byDeck).sort((a, b) => b[1].rate - a[1].rate)) L.push(`| ${k} | ${fmtRate(v)}${sig(v)} |`);
const tags = Object.entries(s.byTag).filter(([, v]) => v.n >= 30).sort((a, b) => b[1].rate - a[1].rate);
if (tags.length) {
L.push('', '| Tag (archetype/faction/keyword) | WR |', '|---|---|');
for (const [k, v] of tags) L.push(`| ${k} | ${fmtRate(v)}${sig(v)} |`);
}
const decks = Object.keys(s.matchups).sort();
if (decks.length <= 12) {
L.push('', `| row vs col | ${decks.join(' | ')} |`, `|---|${decks.map(() => '---').join('|')}|`);
for (const r of decks) L.push(`| ${r} | ${decks.map(c => (s.matchups[r][c] ? pct(s.matchups[r][c].rate) : '')).join(' | ')} |`);
}
const elig = s.cards.filter(c => c.games >= 50);
L.push('', `**Cards** (${s.cards.length} seen, ${elig.length} with ≥50 games; WR = win rate of the side that played it):`, '');
L.push('| Top cards | games | WR |', '|---|---|---|');
for (const c of elig.slice(0, 10)) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`);
L.push('', '| Bottom cards | games | WR |', '|---|---|---|');
for (const c of elig.slice(-10).reverse()) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`);
const corr = Object.entries(s.statCorrelation).filter(([, v]) => v.n >= 10);
if (corr.length) {
L.push('', '**Card stats vs win rate** (numeric: Pearson r across cards; `card k:v`: WR − 50%):', '');
for (const [k, v] of corr.sort((a, b) => Math.abs(b[1].r) - Math.abs(a[1].r))) L.push(`- ${k}: ${v.r >= 0 ? '+' : ''}${v.r.toFixed(3)} (n=${v.n})`);
}
L.push('');
return L.join('\n');
}