Files
wikideck/scripts/sim/stats.ts

218 lines
12 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Aggregate MatchRecords into balance statistics and a markdown section.
import type { EngineAdapter, MatchRecord, Seat } from '../../shared/bots/types.ts';
export interface Rate { n: number; wins: number; rate: number; lo: number; hi: number }
/** Wilson 95% interval. */
export function rate(wins: number, n: number): Rate {
if (!n) return { n, wins, rate: NaN, lo: NaN, hi: NaN };
const z = 1.96, p = wins / n, d = 1 + z * z / n;
const c = (p + z * z / (2 * n)) / d, h = (z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n))) / d;
return { n, wins, rate: p, lo: c - h, hi: c + h };
}
const quant = (xs: number[], q: number) => {
if (!xs.length) return NaN;
const s = [...xs].sort((a, b) => a - b);
return s[Math.min(s.length - 1, Math.floor(q * s.length))];
};
const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / (xs.length || 1);
function pearson(xs: number[], ys: number[]): number {
const mx = mean(xs), my = mean(ys);
let sxy = 0, sxx = 0, syy = 0;
for (let i = 0; i < xs.length; i++) { const dx = xs[i] - mx, dy = ys[i] - my; sxy += dx * dy; sxx += dx * dx; syy += dy * dy; }
return sxx && syy ? sxy / Math.sqrt(sxx * syy) : NaN;
}
/** delta = deck-adjusted lift: mean over games of (won − that deck's overall WR in symmetric games). */
export interface CardStat { id: string; label?: string; games: number; plays: number; wins: number; wr: Rate; delta: number }
export interface RulesetStats {
ruleset: string;
matches: number;
draws: number;
truncated: number;
errors: number;
turns: { mean: number; p10: number; p50: number; p90: number };
firstPlayer: Rate;
early: { decidedMedianFrac: number; decidedByHalf: number; decidedByThird: number };
byBotPair: Record<string, { n: number; seat0: Rate; firstPlayer: Rate; turns: number }>;
skill: Record<string, Rate>; // "lookahead>greedy": win rate of the stronger bot
byDeck: Record<string, Rate>; // excluding mirrors
byTag: Record<string, Rate>; // excluding games where both sides share the tag
matchups: Record<string, Record<string, Rate>>;
cards: CardStat[];
statCorrelation: Record<string, { r: number; n: number }>;
reasons: Record<string, number>;
/** Counterfactual keyword value (see EngineAdapter.keywordValues), per keyword. */
keywords: Record<string, { n: number; cards: number; powerPerCard: number; flipRate: number; wr: Rate }>;
}
/** Only bot-symmetric games are used for deck/tag/card/first-player balance. */
const symmetric = (m: MatchRecord) => m.bots[0] === m.bots[1];
export function aggregate(records: MatchRecord[], errors = 0, engine?: EngineAdapter<any, any>): RulesetStats {
const ruleset = records[0]?.ruleset ?? '?';
const decisive = records.filter(m => m.winner !== null);
const sym = decisive.filter(symmetric);
const reasons: Record<string, number> = {};
for (const m of records) reasons[m.reason ?? 'none'] = (reasons[m.reason ?? 'none'] ?? 0) + 1;
const turns = records.map(m => m.turns);
const fracs = decisive.filter(m => m.decidedTurn !== null && m.turns > 1).map(m => (m.decidedTurn! - 1) / Math.max(1, m.turns - 1));
const byBotPair: RulesetStats['byBotPair'] = {};
const pairs = new Map<string, MatchRecord[]>();
for (const m of records) { const k = `${m.bots[0]} vs ${m.bots[1]}`; pairs.set(k, [...(pairs.get(k) ?? []), m]); }
for (const [k, ms] of pairs) {
const d = ms.filter(m => m.winner !== null);
byBotPair[k] = {
n: ms.length,
seat0: rate(d.filter(m => m.winner === 0).length, d.length),
firstPlayer: rate(d.filter(m => m.winner === m.firstSeat).length, d.length),
turns: mean(ms.map(m => m.turns)),
};
}
const skill: Record<string, Rate> = {};
const order = ['random', 'maxpower', 'greedy', 'lookahead-lite', 'lookahead'];
for (const m of decisive) {
if (symmetric(m)) continue;
const [a, b] = m.bots;
const strong: Seat = order.indexOf(a) >= order.indexOf(b) ? 0 : 1;
const k = `${m.bots[strong]}>${m.bots[1 - strong]}`;
const r = skill[k] ?? (skill[k] = rate(0, 0));
r.n++; if (m.winner === strong) r.wins++;
}
for (const k of Object.keys(skill)) skill[k] = rate(skill[k].wins, skill[k].n);
const tally = (map: Record<string, { w: number; n: number }>, key: string, won: boolean) => {
const t = map[key] ?? (map[key] = { w: 0, n: 0 }); t.n++; if (won) t.w++;
};
const deckT: Record<string, { w: number; n: number }> = {}, tagT: Record<string, { w: number; n: number }> = {};
const mu: Record<string, Record<string, { w: number; n: number }>> = {};
const cardT = new Map<string, { games: number; plays: number; wins: number; lift: number }>();
const deckAll: Record<string, { w: number; n: number }> = {};
for (const m of sym) for (const seat of [0, 1] as Seat[]) tally(deckAll, m.decks[seat], m.winner === seat);
const deckRate = (d: string) => deckAll[d].w / deckAll[d].n;
for (const m of sym) {
for (const seat of [0, 1] as Seat[]) {
const won = m.winner === seat, o = (1 - seat) as Seat;
if (m.decks[0] !== m.decks[1]) tally(deckT, m.decks[seat], won);
for (const t of m.deckTags[seat]) if (!m.deckTags[o].includes(t)) tally(tagT, t, won);
tally(mu[m.decks[seat]] ?? (mu[m.decks[seat]] = {}), m.decks[o], won);
const seen = new Set<string>();
for (const c of m.cardsPlayed[seat]) {
const cs = cardT.get(c) ?? { games: 0, plays: 0, wins: 0, lift: 0 };
cs.plays++;
if (!seen.has(c)) { seen.add(c); cs.games++; if (won) cs.wins++; cs.lift += Number(won) - deckRate(m.decks[seat]); }
cardT.set(c, cs);
}
}
}
const toRates = (t: Record<string, { w: number; n: number }>) => Object.fromEntries(Object.entries(t).map(([k, v]) => [k, rate(v.w, v.n)]));
const cards: CardStat[] = [...cardT].map(([id, c]) => {
const wr = rate(c.wins, c.games);
const label = engine?.cardMeta?.(id)?.label;
return { id, label: typeof label === 'string' ? label : undefined, games: c.games, plays: c.plays, wins: c.wins, wr, delta: c.lift / c.games };
}).sort((a, b) => b.delta - a.delta);
const statCorrelation: RulesetStats['statCorrelation'] = {};
if (engine?.cardMeta) {
const eligible = cards.filter(c => c.games >= 30);
const keys = new Set<string>();
const metas = eligible.map(c => engine.cardMeta!(c.id) ?? {});
for (const m of metas) for (const [k, v] of Object.entries(m)) if (typeof v === 'number') keys.add(k);
for (const k of keys) {
const xs: number[] = [], ys: number[] = [];
eligible.forEach((c, i) => { const v = metas[i][k]; if (typeof v === 'number') { xs.push(v); ys.push(c.delta); } });
statCorrelation[k] = { r: pearson(xs, ys), n: xs.length };
}
// keyword / faction tags on cards
const tagW: Record<string, { w: number; n: number }> = {};
eligible.forEach((c, i) => {
for (const [k, v] of Object.entries(metas[i])) {
const vals = Array.isArray(v) ? v : typeof v === 'string' ? [v] : [];
if (k === 'label') continue;
for (const x of vals) { const t = tagW[`${k}:${x}`] ?? (tagW[`${k}:${x}`] = { w: 0, n: 0 }); t.w += c.delta * c.games; t.n += c.games; }
}
});
for (const [k, v] of Object.entries(tagW)) statCorrelation[`card ${k}`] = { r: v.w / v.n, n: v.n };
}
const kwT: Record<string, { n: number; cards: number; power: number; flip: number; wins: number }> = {};
for (const m of sym) for (const k of m.keywordValues ?? []) {
const t = kwT[k.keyword] ?? (kwT[k.keyword] = { n: 0, cards: 0, power: 0, flip: 0, wins: 0 });
t.n++; t.cards += k.cards; t.power += k.power; t.flip += k.flip; if (m.winner === k.seat) t.wins++;
}
const keywords = Object.fromEntries(Object.entries(kwT).map(([k, t]) => [k, { n: t.n, cards: t.cards, powerPerCard: t.power / t.cards, flipRate: t.flip / t.n, wr: rate(t.wins, t.n) }]));
return {
ruleset, matches: records.length, errors, keywords,
draws: records.filter(m => m.winner === null).length,
truncated: records.filter(m => m.truncated).length,
turns: { mean: mean(turns), p10: quant(turns, 0.1), p50: quant(turns, 0.5), p90: quant(turns, 0.9) },
firstPlayer: rate(sym.filter(m => m.winner === m.firstSeat).length, sym.length),
early: {
decidedMedianFrac: quant(fracs, 0.5),
decidedByHalf: fracs.filter(f => f <= 0.5).length / (fracs.length || 1),
decidedByThird: fracs.filter(f => f <= 1 / 3).length / (fracs.length || 1),
},
byBotPair, skill, byDeck: toRates(deckT), byTag: toRates(tagT),
matchups: Object.fromEntries(Object.entries(mu).map(([k, v]) => [k, toRates(v)])),
cards, statCorrelation, reasons,
};
}
const pct = (x: number) => (Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : 'n/a');
const fmtRate = (r: Rate) => `${pct(r.rate)} [${pct(r.lo)}–${pct(r.hi)}] (n=${r.n})`;
const sig = (r: Rate) => (r.lo > 0.5 ? ' **↑**' : r.hi < 0.5 ? ' **↓**' : '');
export function toMarkdown(s: RulesetStats): string {
const L: string[] = [];
L.push(`### Ruleset \`${s.ruleset}\``, '');
L.push(`- Matches: **${s.matches}** (draws ${s.draws}, hit action cap ${s.truncated}, engine errors ${s.errors})`);
L.push(`- First-player win rate (bot-symmetric games): **${fmtRate(s.firstPlayer)}**${sig(s.firstPlayer)}`);
L.push(`- Match length (turns, both players): mean ${s.turns.mean.toFixed(1)}, p10 ${s.turns.p10}, median ${s.turns.p50}, p90 ${s.turns.p90}`);
L.push(`- Decided early: median game is locked in at ${pct(s.early.decidedMedianFrac)} of its length; ${pct(s.early.decidedByThird)} decided in the first third, ${pct(s.early.decidedByHalf)} by half-time`);
L.push(`- End reasons: ${Object.entries(s.reasons).map(([k, v]) => `${k} ${v}`).join(', ')}`, '');
L.push('| Bot pairing | n | seat-0 WR | first-player WR | mean turns |', '|---|---|---|---|---|');
for (const [k, v] of Object.entries(s.byBotPair)) L.push(`| ${k} | ${v.n} | ${fmtRate(v.seat0)} | ${fmtRate(v.firstPlayer)} | ${v.turns.toFixed(1)} |`);
if (Object.keys(s.skill).length) {
L.push('', '**Skill expression** (stronger bot win rate):', '');
for (const [k, v] of Object.entries(s.skill)) L.push(`- ${k}: ${fmtRate(v)}`);
}
L.push('', '| Deck | WR vs other decks |', '|---|---|');
for (const [k, v] of Object.entries(s.byDeck).sort((a, b) => b[1].rate - a[1].rate)) L.push(`| ${k} | ${fmtRate(v)}${sig(v)} |`);
const tags = Object.entries(s.byTag).filter(([, v]) => v.n >= 30).sort((a, b) => b[1].rate - a[1].rate);
if (tags.length) {
L.push('', '| Tag (archetype/faction/keyword) | WR |', '|---|---|');
for (const [k, v] of tags) L.push(`| ${k} | ${fmtRate(v)}${sig(v)} |`);
}
const decks = Object.keys(s.matchups).sort();
if (decks.length <= 12) {
L.push('', `| row vs col | ${decks.join(' | ')} |`, `|---|${decks.map(() => '---').join('|')}|`);
for (const r of decks) L.push(`| ${r} | ${decks.map(c => (s.matchups[r][c] ? pct(s.matchups[r][c].rate) : '')).join(' | ')} |`);
}
const elig = s.cards.filter(c => c.games >= 50);
L.push('', `**Cards** (${s.cards.length} seen, ${elig.length} with ≥50 games). Lift = win rate when played minus the win rate of the deck it was in (deck-adjusted); raw WR for reference:`, '');
const meta = (c: CardStat) => c.label ?? c.id;
L.push('| Top cards | games | lift | raw WR |', '|---|---|---|---|');
for (const c of elig.slice(0, 10)) L.push(`| ${meta(c)} | ${c.games} | ${c.delta >= 0 ? '+' : ''}${(c.delta * 100).toFixed(1)} pp | ${pct(c.wr.rate)} |`);
L.push('', '| Bottom cards | games | lift | raw WR |', '|---|---|---|---|');
for (const c of elig.slice(-10).reverse()) L.push(`| ${meta(c)} | ${c.games} | ${c.delta >= 0 ? '+' : ''}${(c.delta * 100).toFixed(1)} pp | ${pct(c.wr.rate)} |`);
if (Object.keys(s.keywords).length) {
L.push('', '**Keyword value** (counterfactual replay with the keyword removed from that side; same actions):', '');
L.push('| Keyword | side-games | cards played | power per card | games decided by it | WR of side playing it |', '|---|---|---|---|---|---|');
for (const [k, v] of Object.entries(s.keywords).sort((a, b) => b[1].powerPerCard - a[1].powerPerCard))
L.push(`| ${k} | ${v.n} | ${v.cards} | ${v.powerPerCard >= 0 ? '+' : ''}${v.powerPerCard.toFixed(2)} | ${pct(v.flipRate)} | ${fmtRate(v.wr)}${sig(v.wr)} |`);
}
const corr = Object.entries(s.statCorrelation).filter(([, v]) => v.n >= 10);
if (corr.length) {
L.push('', '**Card stats vs lift** (numeric: Pearson r across cards; `card k:v`: mean lift, as a fraction):', '');
for (const [k, v] of corr.sort((a, b) => Math.abs(b[1].r) - Math.abs(a[1].r))) L.push(`- ${k}: ${v.r >= 0 ? '+' : ''}${v.r.toFixed(3)} (n=${v.n})`);
}
L.push('');
return L.join('\n');
}