Files
wikideck/scripts/sim/duel-stats.ts
T
claudeandClaude Opus 5.5 038e079ce8 [sim-balance] Duel bots (heuristic, MC) and duel simulation harness
Engine-agnostic duel bots over a DuelLens, a 40-card deck builder from the
236 sample cards with type/curve-skewed archetypes, a <=4-worker sim runner
and DUEL_BALANCE metrics. Runs on a small stand-in duel engine until
duel-core's shared/duel alpha lands; then shared/bots/duel-adapter.ts binds it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-17 21:39:48 -05:00

157 lines
9.9 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Aggregate duel match records into the DUEL_BALANCE.md metrics.
import type { Seat } from '../../shared/bots/types.ts';
import { rate, type Rate } from './stats.ts';
import type { SimDeck, PoolCard } from './duel-decks.ts';
import type { DuelSim } from './duel-sim-types.ts';
export interface DuelRecord {
seed: number; bots: [string, string]; decks: [string, string]; tags: [string[], string[]];
first: Seat; winner: Seat | null; reason: string; turns: number; turnsEach: number; actions: number; truncated: boolean;
minutes: number; played: [string[], string[]]; metrics: Record<string, number>; ms: number;
}
const quant = (xs: number[], q: number) => { if (!xs.length) return NaN; const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(q * s.length))]; };
const mean = (xs: number[]) => xs.reduce((a, b) => a + b, 0) / (xs.length || 1);
function pearson(xs: number[], ys: number[]) {
const mx = mean(xs), my = mean(ys); let sxy = 0, sxx = 0, syy = 0;
for (let i = 0; i < xs.length; i++) { const dx = xs[i] - mx, dy = ys[i] - my; sxy += dx * dy; sxx += dx * dx; syy += dy * dy; }
return sxx && syy ? sxy / Math.sqrt(sxx * syy) : NaN;
}
export interface DuelStats {
meta: Record<string, unknown>;
matches: number; errors: number; draws: number; truncated: number; reasons: Record<string, number>;
firstPlayer: Rate; firstPlayerAll: Rate;
turnsEach: { p10: number; p50: number; p90: number; mean: number };
minutes: { p10: number; p50: number; p90: number };
stalls: { actionCap: number; turnCap: number; over20Each: number; rate: number };
skill: Record<string, Rate>;
byTag: Record<string, Rate>;
byType: Record<string, Rate & { plays: number }>;
byPortal: Record<string, Rate & { plays: number }>;
byKeyword: Record<string, Rate & { plays: number }>;
byCost: Record<string, Rate & { plays: number }>;
fame: { rCardWr: number; rWithinCost: Record<string, number>; topDecile: Rate; bottomDecile: Rate; cards: number };
connected: { pairRate: number; pairs: number; firesPerPlay: number; firesPerGame: number };
cards: { id: string; title?: string; type: string; cost: number; fame?: number; games: number; wr: Rate }[];
}
const sym = (m: DuelRecord) => m.bots[0] === m.bots[1];
export function aggregateDuel(records: DuelRecord[], sim: DuelSim<any, any>, decks: SimDeck[], errors: number, meta: Record<string, unknown>): DuelStats {
const byId = new Map<string, PoolCard & { title?: string }>(sim.pool.map((c: any) => [c.id, c]));
const decisive = records.filter(m => m.winner !== null);
const symDec = decisive.filter(sym);
const reasons: Record<string, number> = {};
for (const m of records) reasons[m.reason] = (reasons[m.reason] ?? 0) + 1;
const skill: Record<string, Rate> = {};
const pairs: Record<string, [number, number]> = {};
for (const m of decisive) {
if (sym(m)) continue;
const key = [...m.bots].sort().join('>');
const [a] = key.split('>');
const aSeat = m.bots[0] === a ? 0 : 1;
const p = pairs[key] ?? (pairs[key] = [0, 0]); p[0]++; if (m.winner === aSeat) p[1]++;
}
for (const [k, [n, w]] of Object.entries(pairs)) {
// Report as "stronger>weaker" with the stronger side's rate.
const [a, b] = k.split('>'); const r = rate(w, n);
if (r.rate >= 0.5) skill[`${a}>${b}`] = r; else skill[`${b}>${a}`] = rate(n - w, n);
}
// Tags: exclude games where both decks share the tag.
const tagAcc: Record<string, [number, number]> = {};
for (const m of symDec) for (const s of [0, 1] as Seat[]) for (const t of m.tags[s]) {
if (m.tags[1 - s].includes(t)) continue;
const a = tagAcc[t] ?? (tagAcc[t] = [0, 0]); a[0]++; if (m.winner === s) a[1]++;
}
const byTag = Object.fromEntries(Object.entries(tagAcc).sort().map(([k, [n, w]]) => [k, rate(w, n)]));
// Card-level: a card "appears" in a game if its seat played it. WR of the seat that played it.
const cardAcc = new Map<string, [number, number]>();
for (const m of symDec) for (const s of [0, 1] as Seat[]) for (const id of new Set(m.played[s])) {
if (new Set(m.played[1 - s]).has(id)) continue;
const a = cardAcc.get(id) ?? [0, 0]; a[0]++; if (m.winner === s) a[1]++; cardAcc.set(id, a);
}
const group = (f: (c: PoolCard) => string[]) => {
const acc: Record<string, [number, number]> = {};
for (const [id, [n, w]] of cardAcc) { const c = byId.get(id); if (!c) continue; for (const k of f(c)) { const a = acc[k] ?? (acc[k] = [0, 0]); a[0] += n; a[1] += w; } }
return Object.fromEntries(Object.entries(acc).sort().map(([k, [n, w]]) => [k, { ...rate(w, n), plays: n }]));
};
const cards = [...cardAcc].map(([id, [n, w]]) => { const c = byId.get(id) as any; return { id, title: c?.title, type: c?.type ?? '?', cost: c?.cost ?? 0, fame: c?.fame, games: n, wr: rate(w, n) }; })
.sort((a, b) => b.games - a.games);
const solid = cards.filter(c => c.games >= 20 && typeof c.fame === 'number');
const rWithinCost: Record<string, number> = {};
for (const cost of [1, 2, 3, 4, 5, 6]) { const xs = solid.filter(c => c.cost === cost); if (xs.length >= 5) rWithinCost[cost] = pearson(xs.map(c => c.fame!), xs.map(c => c.wr.rate)); }
const byFame = [...solid].sort((a, b) => a.fame! - b.fame!);
const dec = Math.max(1, Math.floor(byFame.length / 10));
const pool = (xs: typeof solid) => rate(xs.reduce((t, c) => t + c.wr.wins, 0), xs.reduce((t, c) => t + c.games, 0));
// Connected: static pair rate over the pool, and in-game fires.
let pairsN = 0, hits = 0;
if (sim.connected) for (let i = 0; i < sim.pool.length; i++) for (let j = i + 1; j < sim.pool.length; j++) { pairsN++; if (sim.connected(sim.pool[i], sim.pool[j])) hits++; }
const fires = records.map(m => m.metrics.connectedFires ?? 0), plays = records.map(m => m.metrics.plays ?? 0);
const turns = records.filter(m => !m.truncated).map(m => m.turnsEach);
const fp = (xs: DuelRecord[]) => rate(xs.filter(m => m.winner === m.first).length, xs.length);
const actionCap = records.filter(m => m.truncated).length, turnCap = reasons['turn-cap'] ?? 0, over20 = records.filter(m => m.turnsEach > 20).length;
return {
meta: { ...meta, decks: decks.length, when: new Date().toISOString() },
matches: records.length, errors, draws: records.length - decisive.length, truncated: actionCap, reasons,
firstPlayer: fp(symDec), firstPlayerAll: fp(decisive),
turnsEach: { p10: quant(turns, 0.1), p50: quant(turns, 0.5), p90: quant(turns, 0.9), mean: mean(turns) },
minutes: { p10: quant(records.map(m => m.minutes), 0.1), p50: quant(records.map(m => m.minutes), 0.5), p90: quant(records.map(m => m.minutes), 0.9) },
stalls: { actionCap, turnCap, over20Each: over20, rate: (actionCap + turnCap + over20) / (records.length || 1) },
skill, byTag,
byType: group(c => [c.type]), byPortal: group(c => [c.portal]), byKeyword: group(c => (c.keywords.length ? c.keywords : ['(none)'])), byCost: group(c => [String(c.cost)]),
fame: { rCardWr: pearson(solid.map(c => c.fame!), solid.map(c => c.wr.rate)), rWithinCost, topDecile: pool(byFame.slice(-dec)), bottomDecile: pool(byFame.slice(0, dec)), cards: solid.length },
connected: { pairRate: pairsN ? hits / pairsN : NaN, pairs: pairsN, firesPerPlay: mean(plays) ? mean(fires) / mean(plays) : NaN, firesPerGame: mean(fires) },
cards,
};
}
const pc = (r: Rate) => (Number.isNaN(r.rate) ? '—' : `${(100 * r.rate).toFixed(1)}% [${(100 * r.lo).toFixed(0)}–${(100 * r.hi).toFixed(0)}] n=${r.n}`);
const table = (title: string, m: Record<string, Rate & { plays?: number }>) =>
`| ${title} | Win rate (95% CI) |\n|---|---|\n` + Object.entries(m).map(([k, r]) => `| ${k} | ${pc(r)} |`).join('\n') + '\n';
export function duelMarkdown(s: DuelStats): string {
const f = (x: number, d = 1) => (Number.isNaN(x) ? '—' : x.toFixed(d));
const top = s.cards.filter(c => c.games >= 20).sort((a, b) => b.wr.rate - a.wr.rate);
const line = (c: DuelStats['cards'][number]) => `| ${c.title ?? c.id} | ${c.type} | ${c.cost} | ${f(c.fame ?? NaN, 0)} | ${pc(c.wr)} |`;
return `### Run \`${s.meta.label}\` (engine \`${s.meta.engine}\`${s.meta.sha ? `, SHA \`${s.meta.sha}\`` : ''})
Rules overrides: \`${JSON.stringify(s.meta.rules)}\`. ${s.matches} matches, ${s.meta.decks} decks, ${s.errors} errors, ${s.draws} draws.
| Metric | Value | Target |
|---|---|---|
| First-player win rate (symmetric bots) | ${pc(s.firstPlayer)} | 48–52% |
| First-player win rate (all decisive) | ${pc(s.firstPlayerAll)} | |
| Turns each p10 / median / p90 | ${f(s.turnsEach.p10, 0)} / **${f(s.turnsEach.p50, 0)}** / ${f(s.turnsEach.p90, 0)} | median 8–12 |
| Est. minutes p10 / median / p90 | ${f(s.minutes.p10)} / **${f(s.minutes.p50)}** / ${f(s.minutes.p90)} | 8–15 |
| Stalls (action cap + turn cap + >20 turns each) | ${s.stalls.actionCap} + ${s.stalls.turnCap} + ${s.stalls.over20Each} (${f(100 * s.stalls.rate)}%) | ~0 |
| Connected: static pair rate over the pool | ${f(100 * s.connected.pairRate)}% of ${s.connected.pairs} pairs | 15–30% |
| Connected: fires per play / per game | ${f(100 * s.connected.firesPerPlay)}% / ${f(s.connected.firesPerGame, 2)} | |
| Fame vs card win rate (Pearson r, cards with ≥20 games) | ${f(s.fame.rCardWr, 2)} (n=${s.fame.cards}) | ≈0 (not strictly better) |
| Top vs bottom fame decile | ${pc(s.fame.topDecile)} vs ${pc(s.fame.bottomDecile)} | close |
End reasons: ${Object.entries(s.reasons).map(([k, v]) => `${k} ${v}`).join(', ')}.
Skill (stronger bot's win rate): ${Object.entries(s.skill).map(([k, r]) => `${k} ${pc(r)}`).join('; ') || '—'}.
Fame vs win rate within each cost: ${Object.entries(s.fame.rWithinCost).map(([k, r]) => `cost ${k}: r=${f(r, 2)}`).join(', ') || '—'}.
${table('Card type (seat that played it)', s.byType)}
${table('Portal', s.byPortal)}
${table('Keyword', s.byKeyword)}
${table('Cost', s.byCost)}
${table('Deck tag (vs decks without it)', s.byTag)}
Top cards by win rate when played (≥20 games):
| Card | Type | Cost | Fame | WR |
|---|---|---|---|---|
${top.slice(0, 8).map(line).join('\n')}
Bottom cards:
| Card | Type | Cost | Fame | WR |
|---|---|---|---|---|
${top.slice(-8).reverse().map(line).join('\n')}
`;
}