diff --git a/scripts/sim/match.ts b/scripts/sim/match.ts index 05fdca6..ffca574 100644 --- a/scripts/sim/match.ts +++ b/scripts/sim/match.ts @@ -37,6 +37,7 @@ export function playMatch( turns: engine.turn(s), actions, truncated, evalTrace, decidedTurn: decidedTurn(evalTrace, winner), cardsPlayed, actionKinds, + keywordValues: !truncated && engine.keywordValues ? engine.keywordValues(s, decks) : undefined, ms: performance.now() - t0, }; } diff --git a/scripts/sim/run.ts b/scripts/sim/run.ts index 86ebba5..2591a03 100644 --- a/scripts/sim/run.ts +++ b/scripts/sim/run.ts @@ -25,7 +25,7 @@ export async function loadEngine(name: string): Promise> const base = rulesets[parent]; const merged: any = { ...base }; for (const [k, val] of Object.entries(patch)) merged[k] = val && typeof val === 'object' && !Array.isArray(val) ? { ...base[k], ...val } : val; - rulesets[id] = { ...merged, id: v.id ?? `sim-${id}`.replace(/[^a-z0-9-]/g, '-').slice(0, 48) }; + rulesets[id] = merged; } } return makeGameEngine({ rulesets }); @@ -44,6 +44,12 @@ export const DEFAULT_MIX: [string, string, number][] = [ ['random', 'greedy', 0.05], ]; +export const MIXES: Record = { + default: DEFAULT_MIX, + // Fast sweep: balance from greedy mirrors, plus a little skill signal. + fast: [['greedy', 'greedy', 0.8], ['maxpower', 'greedy', 0.1], ['random', 'greedy', 0.1]], +}; + interface Job { i: number; seed: number; decks: [number, number]; bots: [string, string] } export function schedule(n: number, nDecks: number, baseSeed: number, mix = DEFAULT_MIX): Job[] { @@ -84,7 +90,7 @@ async function main() { const sections: string[] = []; for (const ruleset of rulesets) { const decks = engine.decks(ruleset); - const jobs = schedule(n, decks.length, seed); + const jobs = schedule(n, decks.length, seed, MIXES[arg('mix', 'default')!]); const records: MatchRecord[] = []; const errors: string[] = []; const file = createWriteStream(join(out, `${ruleset}.jsonl`)); diff --git a/scripts/sim/stats.ts b/scripts/sim/stats.ts index fae4313..2de2b5d 100644 --- a/scripts/sim/stats.ts +++ b/scripts/sim/stats.ts @@ -44,6 +44,8 @@ export interface RulesetStats { cards: CardStat[]; statCorrelation: Record; reasons: Record; + /** Counterfactual keyword value (see EngineAdapter.keywordValues), per keyword. */ + keywords: Record; } /** Only bot-symmetric games are used for deck/tag/card/first-player balance. */ @@ -134,8 +136,15 @@ export function aggregate(records: MatchRecord[], errors = 0, engine?: EngineAda for (const [k, v] of Object.entries(tagW)) statCorrelation[`card ${k}`] = { r: v.w / v.n - 0.5, n: v.n }; } + const kwT: Record = {}; + for (const m of sym) for (const k of m.keywordValues ?? []) { + const t = kwT[k.keyword] ?? (kwT[k.keyword] = { n: 0, cards: 0, power: 0, flip: 0, wins: 0 }); + t.n++; t.cards += k.cards; t.power += k.power; t.flip += k.flip; if (m.winner === k.seat) t.wins++; + } + const keywords = Object.fromEntries(Object.entries(kwT).map(([k, t]) => [k, { n: t.n, cards: t.cards, powerPerCard: t.power / t.cards, flipRate: t.flip / t.n, wr: rate(t.wins, t.n) }])); + return { - ruleset, matches: records.length, errors, + ruleset, matches: records.length, errors, keywords, draws: records.filter(m => m.winner === null).length, truncated: records.filter(m => m.truncated).length, turns: { mean: mean(turns), p10: quant(turns, 0.1), p50: quant(turns, 0.5), p90: quant(turns, 0.9) }, @@ -187,6 +196,12 @@ export function toMarkdown(s: RulesetStats): string { for (const c of elig.slice(0, 10)) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`); L.push('', '| Bottom cards | games | WR |', '|---|---|---|'); for (const c of elig.slice(-10).reverse()) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`); + if (Object.keys(s.keywords).length) { + L.push('', '**Keyword value** (counterfactual replay with the keyword removed from that side; same actions):', ''); + L.push('| Keyword | side-games | cards played | power per card | games decided by it | WR of side playing it |', '|---|---|---|---|---|---|'); + for (const [k, v] of Object.entries(s.keywords).sort((a, b) => b[1].powerPerCard - a[1].powerPerCard)) + L.push(`| ${k} | ${v.n} | ${v.cards} | ${v.powerPerCard >= 0 ? '+' : ''}${v.powerPerCard.toFixed(2)} | ${pct(v.flipRate)} | ${fmtRate(v.wr)}${sig(v.wr)} |`); + } const corr = Object.entries(s.statCorrelation).filter(([, v]) => v.n >= 10); if (corr.length) { L.push('', '**Card stats vs win rate** (numeric: Pearson r across cards; `card k:v`: WR − 50%):', ''); diff --git a/scripts/sim/summary.py b/scripts/sim/summary.py new file mode 100644 index 0000000..8091ef3 --- /dev/null +++ b/scripts/sim/summary.py @@ -0,0 +1,11 @@ +# Usage: python3 scripts/sim/summary.py -> one-line-per-ruleset summary table (markdown) +import json, glob, sys, os +d = sys.argv[1] +print('| ruleset | n | draws | first-player WR | heavy | swarm | curve | starters | greedy>maxpower | decided in 1st third |') +print('|---|---|---|---|---|---|---|---|---|---|') +for f in sorted(glob.glob(os.path.join(d, '*.stats.json'))): + s = json.load(open(f)); by = s['byDeck'] + g = lambda k: f"{by[k]['rate']*100:.1f}%" if k in by else 'n/a' + sk = s['skill'].get('greedy>maxpower', {}).get('rate') + fp = s['firstPlayer'] + print(f"| {s['ruleset']} | {s['matches']} | {s['draws']} | {fp['rate']*100:.1f}% [{fp['lo']*100:.1f}–{fp['hi']*100:.1f}] | {g('heavy')} | {g('swarm')} | {g('curve')} | {s['byTag'].get('arch:starter',{}).get('rate',0)*100:.1f}% | {sk*100 if sk else float('nan'):.1f}% | {s['early']['decidedByThird']*100:.1f}% |") diff --git a/scripts/sim/variants/design-sweep.json b/scripts/sim/variants/design-sweep.json new file mode 100644 index 0000000..14996fb --- /dev/null +++ b/scripts/sim/variants/design-sweep.json @@ -0,0 +1,8 @@ +{ + "v1-power": { "powerByCost": [0, 1, 2, 4, 6, 8, 10] }, + "v1-power-cap3": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false }, + "v1-cap3-ot1": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "onTopicBonus": 1 }, + "v1-heavy-buff": { "powerByCost": [0, 1, 2, 4, 6, 9, 12], "laneCapacity": 3, "underdog": false }, + "v1-energy-plus1":{ "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "energyByTurn": [2, 3, 4, 5, 6, 7] }, + "v1-turns7": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "turns": 7, "energyByTurn": [1, 2, 3, 4, 5, 6, 7] } +} diff --git a/shared/bots/game-adapter.ts b/shared/bots/game-adapter.ts index ebe928c..3458458 100644 --- a/shared/bots/game-adapter.ts +++ b/shared/bots/game-adapter.ts @@ -1,9 +1,8 @@ // EngineAdapter over the public shared/game API, plus the starter/archetype decks used // by the balance simulation. Pure: rulesets and extra decks are passed in, no I/O. import { - DEFAULT_RULESET, STARTER_POOL, applyAction, createMatch, isOver, lanePower, - legalActions, result, starterCard, starterDeck, validateDeck, - type Action, type CardInPlay, type GameCard, type MatchState, type PlayerId, type Ruleset, + DEFAULT_RULESET, applyAction, createMatch, isOver, lanePower, legalActions, previewPool, replay, result, starterDeck, validateDeck, + type Action, type CardInPlay, type GameCard, type MatchState, type PlayerId, type PlayerState, type Ruleset, } from '../game/index.ts'; import type { DeckSpec, EngineAdapter, Rng, Seat } from './types.ts'; @@ -15,10 +14,10 @@ const other = (p: PlayerId): PlayerId => (p === 0 ? 1 : 0); */ export function projectedLanes(state: MatchState, seat: Seat): [number, number][] { const me = state.players[seat]; - if (!me.staged.length) return state.lanes.map((_, i) => orient(lanePower(state, i), seat)); + if (!me.queue.length) return state.lanes.map((_, i) => orient(lanePower(state, i), seat)); const lanes = state.lanes.map(l => ({ ...l, cards: [[...l.cards[0]], [...l.cards[1]]] as [CardInPlay[], CardInPlay[]] })); - for (const play of me.staged) { - const card = me.hand.find(c => c.id === play.cardId); + for (const play of me.queue) { + const card = me.hand.find(c => c.cardId === play.cardId); if (card) lanes[play.lane].cards[seat].push({ card, owner: seat, lane: play.lane, turn: state.turn, bonus: 0 }); } const view = { ...state, lanes } as MatchState; @@ -34,25 +33,21 @@ export function evaluateFor(state: MatchState, seat: Seat): number { let lane = 0, total = 0; for (const [a, b] of lanes) { lane += Math.tanh((a - b) / scale); total += a - b; } const me = state.players[seat], them = state.players[other(seat)]; - const handEdge = (me.hand.length - me.staged.length - (them.hand.length - them.staged.length)) * 0.02; - const unspent = me.energy * -0.01; // nudge greedy to use its energy + const handEdge = (me.hand.length - me.queue.length - (them.hand.length - them.queue.length)) * 0.02; + const unspent = energyLeft(me) * -0.01; // nudge greedy to use its energy const v = (lane + Math.tanh(total / 20) * 0.3) / (state.lanes.length + 0.3) + handEdge + unspent; return Math.max(-1, Math.min(1, v)); } /** - * Determinize hidden information before a lookahead for `seat`: the opponent's staged plays + * Determinize hidden information before a lookahead for `seat`: the opponent's queued plays * are taken back (it re-decides in the rollouts) and its hand and deck are reshuffled together. */ export function determinize(state: MatchState, seat: Seat, rng: Rng): MatchState { const s = structuredClone(state); const o = other(seat), opp = s.players[o]; - if (opp.staged.length || opp.ended) { - const refund = opp.staged.reduce((t, p) => t + (opp.hand.find(c => c.id === p.cardId)?.cost ?? 0), 0); - opp.energy += refund; - opp.staged = []; - opp.ended = false; - } + opp.queue = []; + opp.committed = false; const pool = [...opp.hand, ...opp.deck]; for (let i = pool.length - 1; i > 0; i--) { const j = Math.floor(rng() * (i + 1)); [pool[i], pool[j]] = [pool[j], pool[i]]; } opp.hand = pool.slice(0, opp.hand.length); @@ -62,14 +57,25 @@ export function determinize(state: MatchState, seat: Seat, rng: Rng): MatchState return s; } +/** Energy left after queued plays (the engine deducts queued costs at reveal). */ +export function energyLeft(p: PlayerState): number { + return p.energy - p.queue.reduce((t, q) => t + (p.hand.find(c => c.cardId === q.cardId)?.cost ?? 0), 0); +} + // ---------- decks ---------- +/** DESIGN §10.1: cost-1 cards have no keyword and get +vanillaBonus (if the ruleset has that key). */ +function vanillaBonus(rules: Ruleset, c: GameCard): number { + const v = (rules as unknown as Record).vanillaBonus; + return typeof v === 'number' && c.cost === 1 ? v : 0; +} + function tagDeck(cards: GameCard[], base: string[]): string[] { const tags = new Set(base); const cats = new Map(); const kws = new Map(); for (const c of cards) { - cats.set(c.category, (cats.get(c.category) ?? 0) + 1); + cats.set(c.faction, (cats.get(c.faction) ?? 0) + 1); kws.set(c.keyword ?? 'vanilla', (kws.get(c.keyword ?? 'vanilla') ?? 0) + 1); } const [topCat, nCat] = [...cats].sort((a, b) => b[1] - a[1])[0]; @@ -80,13 +86,15 @@ function tagDeck(cards: GameCard[], base: string[]): string[] { return [...tags]; } -/** Base (one copy) cards of the default sim pool: the engine's starter pool. */ +/** Base (one copy) cards of the default sim pool: the engine's preview pool. */ export function starterPoolCards(): GameCard[] { - return STARTER_POOL.map(spec => { const c = starterCard(spec, 0); return { ...c, id: c.id.replace(/:\d+$/, '') }; }); + return previewPool().map(c => ({ ...c, cardId: baseId(c.cardId) })); } -const copyOf = (c: GameCard, copy: number): GameCard => ({ ...c, id: `${c.id}:${copy}` }); -const pageKey = (c: GameCard) => (c.pageId ?? c.id); +/** Card id without its copy suffix (`loaner:123:1` -> `loaner:123`). */ +export const baseId = (id: string) => id.replace(/:\d+$/, ''); +const copyOf = (c: GameCard, copy: number): GameCard => ({ ...c, cardId: `${c.cardId}:${copy}` }); +const pageKey = (c: GameCard) => c.pageId; /** * Fill a deck of `size` from `pool`, best `score` first (score <= 0 = excluded, ties: higher power), @@ -108,8 +116,8 @@ export function buildDeck(pool: GameCard[], score: (c: GameCard) => number, size } /** Deck that follows a target cost curve (counts per cost 1..6), using `filter`ed cards first. */ -function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: GameCard) => boolean = () => true): GameCard[] | null { - const target = [0, 2, 3, 3, 2, 1, 1].map(n => Math.round((n * size) / 12)); +function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: GameCard) => boolean = () => true, curve = [0, 2, 3, 3, 2, 1, 1]): GameCard[] | null { + const target = curve.map(n => Math.round((n * size) / 12)); const out: GameCard[] = []; const used = new Map(); const take = (c: GameCard) => { const k = pageKey(c), n = used.get(k) ?? 0; if (n >= copies) return false; used.set(k, n + 1); out.push(copyOf(c, n)); return true; }; @@ -128,18 +136,23 @@ function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: G * curve, heavy (cost >= 5 first), swarm (cost <= 2), mono-faction and keyword-focused curve decks. */ export function defaultDecks(rules: Ruleset, extra: DeckSpec[] = [], pool: GameCard[] = starterPoolCards()): DeckSpec[] { - const size = rules.deck.min, copies = rules.deck.maxCopiesPerPage; + const size = rules.deckSize, copies = rules.maxCopiesPerPage; const decks: DeckSpec[] = []; + // DESIGN §1.3: power depends only on cost, so loaners follow the ruleset's powerByCost. + const repower = (c: GameCard): GameCard => (c.source === 'loaner' && rules.powerByCost[c.cost] !== undefined ? { ...c, power: rules.powerByCost[c.cost] + (c.keyword ? 0 : vanillaBonus(rules, c)) } : c); + pool = pool.map(repower); const add = (id: string, cards: GameCard[] | null, tags: string[]) => { + if (cards) cards = cards.map(repower); if (cards && !validateDeck(rules, cards).length) decks.push({ id, tags: tagDeck(cards, tags), payload: cards }); }; - for (let i = 0; i < 6; i++) add(`starter-${i}`, starterDeck(`sim-${i}`, rules), ['arch:starter']); + for (let i = 0; i < 6; i++) add(`starter-${i}`, starterDeck(`sim-${i}`), ['arch:starter']); add('curve', curveDeck(pool, size, copies), ['arch:curve']); add('heavy', buildDeck(pool, c => (c.cost >= 5 ? 10 + c.cost : c.cost >= 4 ? 5 : 0), size, copies), ['arch:heavy']); + add('top-heavy', curveDeck(pool, size, copies, undefined, [0, 1, 1, 2, 2, 3, 3]), ['arch:top-heavy']); add('swarm', buildDeck(pool, c => (c.cost <= 2 ? 10 - c.cost : c.cost === 3 ? 1 : 0), size, copies), ['arch:swarm']); - const cats = [...new Set(pool.map(c => c.category))].sort(); - for (const cat of cats) if (pool.filter(c => c.category === cat).length >= 6) - add(`mono-${cat}`, curveDeck(pool, size, copies, c => c.category === cat), ['arch:mono-faction']); + const cats = [...new Set(pool.map(c => c.faction))].sort(); + for (const cat of cats) if (pool.filter(c => c.faction === cat).length >= 6) + add(`mono-${cat}`, curveDeck(pool, size, copies, c => c.faction === cat), ['arch:mono-faction']); const kws = [...new Set(pool.map(c => c.keyword).filter((k): k is string => !!k))].sort(); for (const kw of kws) if (pool.filter(c => c.keyword === kw).length >= 3) add(`kw-${kw}`, curveDeck(pool, size, copies, c => c.keyword === kw), ['arch:keyword']); @@ -171,31 +184,53 @@ export function makeGameEngine(opts: GameEngineOptions = {}): EngineAdapter createMatch(rules(id), [decks[0].payload as GameCard[], decks[1].payload as GameCard[]], `sim-${seed}`), - // Simultaneous turns: seat 0 stages first, then seat 1. Bots never read the other side's staged plays + // Simultaneous turns: seat 0 queues first, then seat 1. Bots never read the other side's queued plays // (evaluateFor hides them; lookahead determinizes), so the order leaks nothing. - toAct: s => (s.players[0].ended ? 1 : 0), - // Unstage is excluded: it never helps a bot and lets random bots loop. - legalActions: (s, seat) => legalActions(s, seat).filter(a => a.type !== 'unstage'), + toAct: s => (s.players[0].committed ? 1 : 0), + // Undo/timeout/concede are excluded: they never help a bot and let random bots loop. + legalActions: (s, seat) => legalActions(s, seat).filter(a => a.type === 'play' || a.type === 'commit'), applyAction, isOver, result: s => { const r = result(s); return { winner: r?.winner ?? null, reason: r?.reason }; }, turn: s => s.turn, evaluate: evaluateFor, - cardsOf: (_s, a) => (a.type === 'play' ? [a.cardId.replace(/:\d+$/, '')] : []), + cardsOf: (_s, a) => (a.type === 'play' ? [baseId(a.cardId)] : []), actionKind: a => a.type, cardMeta(id) { - const c = meta.get(id) ?? meta.get(`${id}:0`); + const c = meta.get(id); if (!c) return undefined; - return { label: c.name, cost: c.cost, power: c.power, powerPerCost: c.power / Math.max(1, c.cost), keyword: c.keyword ?? 'vanilla', category: c.category }; + return { label: c.title, cost: c.cost, power: c.power, powerPerCost: c.power / Math.max(1, c.cost), keyword: c.keyword ?? 'vanilla', faction: c.faction, home: c.home }; }, - firstSeat: s => s.firstToReveal, - actionPower: (s, a) => (a.type === 'play' ? s.players[a.player].hand.find(c => c.id === a.cardId)?.power ?? 0 : -1), + firstSeat: s => s.initiative, + keywordValues(final, decks) { + const out: { seat: Seat; keyword: string; power: number; flip: number; cards: number }[] = []; + const r0 = final.result; + if (!r0) return out; + const payload = [decks[0].payload as GameCard[], decks[1].payload as GameCard[]] as const; + const score = (w: PlayerId | null, seat: Seat) => (w === null ? 0 : w === seat ? 1 : -1); + for (const seat of [0, 1] as Seat[]) { + const played = new Set(final.actions.filter(a => a.type === 'play' && a.player === seat).map(a => (a as { cardId: string }).cardId)); + const byKw = new Map(); + for (const c of payload[seat]) if (c.keyword && played.has(c.cardId)) byKw.set(c.keyword, (byKw.get(c.keyword) ?? 0) + 1); + for (const [kw, cards] of byKw) { + const stripped = payload[seat].map(c => (c.keyword === kw ? { ...c, keyword: null } : c)); + const decks2 = (seat === 0 ? [stripped, payload[1]] : [payload[0], stripped]) as [GameCard[], GameCard[]]; + try { + const r1 = replay(final.ruleset, decks2, final.seed, final.actions).result; + if (!r1) continue; + out.push({ seat, keyword: kw, cards, power: r0.totalPower[seat] - r1.totalPower[seat], flip: score(r0.winner, seat) > score(r1.winner, seat) ? 1 : 0 }); + } catch { /* replay diverged (e.g. a keyword changed legality); skip */ } + } + } + return out; + }, + actionPower: (s, a) => (a.type === 'play' ? s.players[a.player].hand.find(c => c.cardId === a.cardId)?.power ?? 0 : -1), determinize, }; } diff --git a/shared/bots/types.ts b/shared/bots/types.ts index c000d8c..f22a980 100644 --- a/shared/bots/types.ts +++ b/shared/bots/types.ts @@ -44,6 +44,12 @@ export interface EngineAdapter { cardMeta?(cardId: string): Record | undefined; /** Printed power an action puts on the board (-1 for non-plays). Used by the naive maxpower bot. */ actionPower?(state: S, action: A): number; + /** + * Counterfactual keyword value for a finished match: for each (seat, keyword) on cards that seat played, + * replay the same actions with that keyword removed from that seat's cards. `power` = own final total power + * lost, `flip` = 1 if the result got worse for that seat (win->draw/loss, draw->loss), `cards` = cards played. + */ + keywordValues?(final: S, decks: [DeckSpec, DeckSpec]): { seat: Seat; keyword: string; power: number; flip: number; cards: number }[]; /** Seat with the first-player advantage (defaults to toAct at match start). */ firstSeat?(state: S): Seat; /** Resample information hidden from `seat` (opponent hand/staged plays, deck order) before searching. */ @@ -70,6 +76,7 @@ export interface MatchRecord { /** Turn from which the eventual winner led on eval until the end (null for draws). */ decidedTurn: number | null; cardsPlayed: [string[], string[]]; + keywordValues?: { seat: Seat; keyword: string; power: number; flip: number; cards: number }[]; actionKinds: Record; ms: number; } diff --git a/tests/bots-sim.test.ts b/tests/bots-sim.test.ts index 93aacd4..9df3e4d 100644 --- a/tests/bots-sim.test.ts +++ b/tests/bots-sim.test.ts @@ -33,10 +33,10 @@ test('stats helpers', () => { test('bots only return legal actions on shared/game', { skip: !existsSync(new URL('../shared/game/index.ts', import.meta.url)) }, async () => { const { gameEngine } = await import('../shared/bots/game-adapter.ts'); const engine = gameEngine as EngineAdapter; - const decks = engine.decks('base'); + const decks = engine.decks(engine.rulesets()[0]); for (const name of ['random', 'greedy', 'lookahead-lite']) { const bot = makeBot(name, engine); - let s = engine.createMatch('base', [decks[0], decks[decks.length - 1]], 7); + let s = engine.createMatch(engine.rulesets()[0], [decks[0], decks[decks.length - 1]], 7); const rng = mulberry32(1); let steps = 0; while (!engine.isOver(s) && steps++ < 500) {