sim-balance: port to Edit War engine API, keyword counterfactual values, ruleset sweeps

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-16 23:07:29 -05:00
co-authored by Claude Opus 5.5
parent a70e0c57aa
commit 72372bd82f
8 changed files with 125 additions and 42 deletions
+1
View File
@@ -37,6 +37,7 @@ export function playMatch<S, A>(
turns: engine.turn(s), actions, truncated,
evalTrace, decidedTurn: decidedTurn(evalTrace, winner),
cardsPlayed, actionKinds,
keywordValues: !truncated && engine.keywordValues ? engine.keywordValues(s, decks) : undefined,
ms: performance.now() - t0,
};
}
+8 -2
View File
@@ -25,7 +25,7 @@ export async function loadEngine(name: string): Promise<EngineAdapter<any, any>>
const base = rulesets[parent];
const merged: any = { ...base };
for (const [k, val] of Object.entries(patch)) merged[k] = val && typeof val === 'object' && !Array.isArray(val) ? { ...base[k], ...val } : val;
rulesets[id] = { ...merged, id: v.id ?? `sim-${id}`.replace(/[^a-z0-9-]/g, '-').slice(0, 48) };
rulesets[id] = merged;
}
}
return makeGameEngine({ rulesets });
@@ -44,6 +44,12 @@ export const DEFAULT_MIX: [string, string, number][] = [
['random', 'greedy', 0.05],
];
export const MIXES: Record<string, [string, string, number][]> = {
default: DEFAULT_MIX,
// Fast sweep: balance from greedy mirrors, plus a little skill signal.
fast: [['greedy', 'greedy', 0.8], ['maxpower', 'greedy', 0.1], ['random', 'greedy', 0.1]],
};
interface Job { i: number; seed: number; decks: [number, number]; bots: [string, string] }
export function schedule(n: number, nDecks: number, baseSeed: number, mix = DEFAULT_MIX): Job[] {
@@ -84,7 +90,7 @@ async function main() {
const sections: string[] = [];
for (const ruleset of rulesets) {
const decks = engine.decks(ruleset);
const jobs = schedule(n, decks.length, seed);
const jobs = schedule(n, decks.length, seed, MIXES[arg('mix', 'default')!]);
const records: MatchRecord[] = [];
const errors: string[] = [];
const file = createWriteStream(join(out, `${ruleset}.jsonl`));
+16 -1
View File
@@ -44,6 +44,8 @@ export interface RulesetStats {
cards: CardStat[];
statCorrelation: Record<string, { r: number; n: number }>;
reasons: Record<string, number>;
/** Counterfactual keyword value (see EngineAdapter.keywordValues), per keyword. */
keywords: Record<string, { n: number; cards: number; powerPerCard: number; flipRate: number; wr: Rate }>;
}
/** Only bot-symmetric games are used for deck/tag/card/first-player balance. */
@@ -134,8 +136,15 @@ export function aggregate(records: MatchRecord[], errors = 0, engine?: EngineAda
for (const [k, v] of Object.entries(tagW)) statCorrelation[`card ${k}`] = { r: v.w / v.n - 0.5, n: v.n };
}
const kwT: Record<string, { n: number; cards: number; power: number; flip: number; wins: number }> = {};
for (const m of sym) for (const k of m.keywordValues ?? []) {
const t = kwT[k.keyword] ?? (kwT[k.keyword] = { n: 0, cards: 0, power: 0, flip: 0, wins: 0 });
t.n++; t.cards += k.cards; t.power += k.power; t.flip += k.flip; if (m.winner === k.seat) t.wins++;
}
const keywords = Object.fromEntries(Object.entries(kwT).map(([k, t]) => [k, { n: t.n, cards: t.cards, powerPerCard: t.power / t.cards, flipRate: t.flip / t.n, wr: rate(t.wins, t.n) }]));
return {
ruleset, matches: records.length, errors,
ruleset, matches: records.length, errors, keywords,
draws: records.filter(m => m.winner === null).length,
truncated: records.filter(m => m.truncated).length,
turns: { mean: mean(turns), p10: quant(turns, 0.1), p50: quant(turns, 0.5), p90: quant(turns, 0.9) },
@@ -187,6 +196,12 @@ export function toMarkdown(s: RulesetStats): string {
for (const c of elig.slice(0, 10)) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`);
L.push('', '| Bottom cards | games | WR |', '|---|---|---|');
for (const c of elig.slice(-10).reverse()) L.push(`| ${c.label ?? c.id} | ${c.games} | ${fmtRate(c.wr)}${sig(c.wr)} |`);
if (Object.keys(s.keywords).length) {
L.push('', '**Keyword value** (counterfactual replay with the keyword removed from that side; same actions):', '');
L.push('| Keyword | side-games | cards played | power per card | games decided by it | WR of side playing it |', '|---|---|---|---|---|---|');
for (const [k, v] of Object.entries(s.keywords).sort((a, b) => b[1].powerPerCard - a[1].powerPerCard))
L.push(`| ${k} | ${v.n} | ${v.cards} | ${v.powerPerCard >= 0 ? '+' : ''}${v.powerPerCard.toFixed(2)} | ${pct(v.flipRate)} | ${fmtRate(v.wr)}${sig(v.wr)} |`);
}
const corr = Object.entries(s.statCorrelation).filter(([, v]) => v.n >= 10);
if (corr.length) {
L.push('', '**Card stats vs win rate** (numeric: Pearson r across cards; `card k:v`: WR − 50%):', '');
+11
View File
@@ -0,0 +1,11 @@
# Usage: python3 scripts/sim/summary.py <out-dir> -> one-line-per-ruleset summary table (markdown)
import json, glob, sys, os
d = sys.argv[1]
print('| ruleset | n | draws | first-player WR | heavy | swarm | curve | starters | greedy>maxpower | decided in 1st third |')
print('|---|---|---|---|---|---|---|---|---|---|')
for f in sorted(glob.glob(os.path.join(d, '*.stats.json'))):
s = json.load(open(f)); by = s['byDeck']
g = lambda k: f"{by[k]['rate']*100:.1f}%" if k in by else 'n/a'
sk = s['skill'].get('greedy>maxpower', {}).get('rate')
fp = s['firstPlayer']
print(f"| {s['ruleset']} | {s['matches']} | {s['draws']} | {fp['rate']*100:.1f}% [{fp['lo']*100:.1f}–{fp['hi']*100:.1f}] | {g('heavy')} | {g('swarm')} | {g('curve')} | {s['byTag'].get('arch:starter',{}).get('rate',0)*100:.1f}% | {sk*100 if sk else float('nan'):.1f}% | {s['early']['decidedByThird']*100:.1f}% |")
+8
View File
@@ -0,0 +1,8 @@
{
"v1-power": { "powerByCost": [0, 1, 2, 4, 6, 8, 10] },
"v1-power-cap3": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false },
"v1-cap3-ot1": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "onTopicBonus": 1 },
"v1-heavy-buff": { "powerByCost": [0, 1, 2, 4, 6, 9, 12], "laneCapacity": 3, "underdog": false },
"v1-energy-plus1":{ "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "energyByTurn": [2, 3, 4, 5, 6, 7] },
"v1-turns7": { "powerByCost": [0, 1, 2, 4, 6, 8, 10], "laneCapacity": 3, "underdog": false, "turns": 7, "energyByTurn": [1, 2, 3, 4, 5, 6, 7] }
}
+72 -37
View File
@@ -1,9 +1,8 @@
// EngineAdapter over the public shared/game API, plus the starter/archetype decks used
// by the balance simulation. Pure: rulesets and extra decks are passed in, no I/O.
import {
DEFAULT_RULESET, STARTER_POOL, applyAction, createMatch, isOver, lanePower,
legalActions, result, starterCard, starterDeck, validateDeck,
type Action, type CardInPlay, type GameCard, type MatchState, type PlayerId, type Ruleset,
DEFAULT_RULESET, applyAction, createMatch, isOver, lanePower, legalActions, previewPool, replay, result, starterDeck, validateDeck,
type Action, type CardInPlay, type GameCard, type MatchState, type PlayerId, type PlayerState, type Ruleset,
} from '../game/index.ts';
import type { DeckSpec, EngineAdapter, Rng, Seat } from './types.ts';
@@ -15,10 +14,10 @@ const other = (p: PlayerId): PlayerId => (p === 0 ? 1 : 0);
*/
export function projectedLanes(state: MatchState, seat: Seat): [number, number][] {
const me = state.players[seat];
if (!me.staged.length) return state.lanes.map((_, i) => orient(lanePower(state, i), seat));
if (!me.queue.length) return state.lanes.map((_, i) => orient(lanePower(state, i), seat));
const lanes = state.lanes.map(l => ({ ...l, cards: [[...l.cards[0]], [...l.cards[1]]] as [CardInPlay[], CardInPlay[]] }));
for (const play of me.staged) {
const card = me.hand.find(c => c.id === play.cardId);
for (const play of me.queue) {
const card = me.hand.find(c => c.cardId === play.cardId);
if (card) lanes[play.lane].cards[seat].push({ card, owner: seat, lane: play.lane, turn: state.turn, bonus: 0 });
}
const view = { ...state, lanes } as MatchState;
@@ -34,25 +33,21 @@ export function evaluateFor(state: MatchState, seat: Seat): number {
let lane = 0, total = 0;
for (const [a, b] of lanes) { lane += Math.tanh((a - b) / scale); total += a - b; }
const me = state.players[seat], them = state.players[other(seat)];
const handEdge = (me.hand.length - me.staged.length - (them.hand.length - them.staged.length)) * 0.02;
const unspent = me.energy * -0.01; // nudge greedy to use its energy
const handEdge = (me.hand.length - me.queue.length - (them.hand.length - them.queue.length)) * 0.02;
const unspent = energyLeft(me) * -0.01; // nudge greedy to use its energy
const v = (lane + Math.tanh(total / 20) * 0.3) / (state.lanes.length + 0.3) + handEdge + unspent;
return Math.max(-1, Math.min(1, v));
}
/**
* Determinize hidden information before a lookahead for `seat`: the opponent's staged plays
* Determinize hidden information before a lookahead for `seat`: the opponent's queued plays
* are taken back (it re-decides in the rollouts) and its hand and deck are reshuffled together.
*/
export function determinize(state: MatchState, seat: Seat, rng: Rng): MatchState {
const s = structuredClone(state);
const o = other(seat), opp = s.players[o];
if (opp.staged.length || opp.ended) {
const refund = opp.staged.reduce((t, p) => t + (opp.hand.find(c => c.id === p.cardId)?.cost ?? 0), 0);
opp.energy += refund;
opp.staged = [];
opp.ended = false;
}
opp.queue = [];
opp.committed = false;
const pool = [...opp.hand, ...opp.deck];
for (let i = pool.length - 1; i > 0; i--) { const j = Math.floor(rng() * (i + 1)); [pool[i], pool[j]] = [pool[j], pool[i]]; }
opp.hand = pool.slice(0, opp.hand.length);
@@ -62,14 +57,25 @@ export function determinize(state: MatchState, seat: Seat, rng: Rng): MatchState
return s;
}
/** Energy left after queued plays (the engine deducts queued costs at reveal). */
export function energyLeft(p: PlayerState): number {
return p.energy - p.queue.reduce((t, q) => t + (p.hand.find(c => c.cardId === q.cardId)?.cost ?? 0), 0);
}
// ---------- decks ----------
/** DESIGN §10.1: cost-1 cards have no keyword and get +vanillaBonus (if the ruleset has that key). */
function vanillaBonus(rules: Ruleset, c: GameCard): number {
const v = (rules as unknown as Record<string, unknown>).vanillaBonus;
return typeof v === 'number' && c.cost === 1 ? v : 0;
}
function tagDeck(cards: GameCard[], base: string[]): string[] {
const tags = new Set(base);
const cats = new Map<string, number>();
const kws = new Map<string, number>();
for (const c of cards) {
cats.set(c.category, (cats.get(c.category) ?? 0) + 1);
cats.set(c.faction, (cats.get(c.faction) ?? 0) + 1);
kws.set(c.keyword ?? 'vanilla', (kws.get(c.keyword ?? 'vanilla') ?? 0) + 1);
}
const [topCat, nCat] = [...cats].sort((a, b) => b[1] - a[1])[0];
@@ -80,13 +86,15 @@ function tagDeck(cards: GameCard[], base: string[]): string[] {
return [...tags];
}
/** Base (one copy) cards of the default sim pool: the engine's starter pool. */
/** Base (one copy) cards of the default sim pool: the engine's preview pool. */
export function starterPoolCards(): GameCard[] {
return STARTER_POOL.map(spec => { const c = starterCard(spec, 0); return { ...c, id: c.id.replace(/:\d+$/, '') }; });
return previewPool().map(c => ({ ...c, cardId: baseId(c.cardId) }));
}
const copyOf = (c: GameCard, copy: number): GameCard => ({ ...c, id: `${c.id}:${copy}` });
const pageKey = (c: GameCard) => (c.pageId ?? c.id);
/** Card id without its copy suffix (`loaner:123:1` -> `loaner:123`). */
export const baseId = (id: string) => id.replace(/:\d+$/, '');
const copyOf = (c: GameCard, copy: number): GameCard => ({ ...c, cardId: `${c.cardId}:${copy}` });
const pageKey = (c: GameCard) => c.pageId;
/**
* Fill a deck of `size` from `pool`, best `score` first (score <= 0 = excluded, ties: higher power),
@@ -108,8 +116,8 @@ export function buildDeck(pool: GameCard[], score: (c: GameCard) => number, size
}
/** Deck that follows a target cost curve (counts per cost 1..6), using `filter`ed cards first. */
function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: GameCard) => boolean = () => true): GameCard[] | null {
const target = [0, 2, 3, 3, 2, 1, 1].map(n => Math.round((n * size) / 12));
function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: GameCard) => boolean = () => true, curve = [0, 2, 3, 3, 2, 1, 1]): GameCard[] | null {
const target = curve.map(n => Math.round((n * size) / 12));
const out: GameCard[] = [];
const used = new Map<string | number, number>();
const take = (c: GameCard) => { const k = pageKey(c), n = used.get(k) ?? 0; if (n >= copies) return false; used.set(k, n + 1); out.push(copyOf(c, n)); return true; };
@@ -128,18 +136,23 @@ function curveDeck(pool: GameCard[], size: number, copies: number, filter: (c: G
* curve, heavy (cost >= 5 first), swarm (cost <= 2), mono-faction and keyword-focused curve decks.
*/
export function defaultDecks(rules: Ruleset, extra: DeckSpec[] = [], pool: GameCard[] = starterPoolCards()): DeckSpec[] {
const size = rules.deck.min, copies = rules.deck.maxCopiesPerPage;
const size = rules.deckSize, copies = rules.maxCopiesPerPage;
const decks: DeckSpec[] = [];
// DESIGN §1.3: power depends only on cost, so loaners follow the ruleset's powerByCost.
const repower = (c: GameCard): GameCard => (c.source === 'loaner' && rules.powerByCost[c.cost] !== undefined ? { ...c, power: rules.powerByCost[c.cost] + (c.keyword ? 0 : vanillaBonus(rules, c)) } : c);
pool = pool.map(repower);
const add = (id: string, cards: GameCard[] | null, tags: string[]) => {
if (cards) cards = cards.map(repower);
if (cards && !validateDeck(rules, cards).length) decks.push({ id, tags: tagDeck(cards, tags), payload: cards });
};
for (let i = 0; i < 6; i++) add(`starter-${i}`, starterDeck(`sim-${i}`, rules), ['arch:starter']);
for (let i = 0; i < 6; i++) add(`starter-${i}`, starterDeck(`sim-${i}`), ['arch:starter']);
add('curve', curveDeck(pool, size, copies), ['arch:curve']);
add('heavy', buildDeck(pool, c => (c.cost >= 5 ? 10 + c.cost : c.cost >= 4 ? 5 : 0), size, copies), ['arch:heavy']);
add('top-heavy', curveDeck(pool, size, copies, undefined, [0, 1, 1, 2, 2, 3, 3]), ['arch:top-heavy']);
add('swarm', buildDeck(pool, c => (c.cost <= 2 ? 10 - c.cost : c.cost === 3 ? 1 : 0), size, copies), ['arch:swarm']);
const cats = [...new Set(pool.map(c => c.category))].sort();
for (const cat of cats) if (pool.filter(c => c.category === cat).length >= 6)
add(`mono-${cat}`, curveDeck(pool, size, copies, c => c.category === cat), ['arch:mono-faction']);
const cats = [...new Set(pool.map(c => c.faction))].sort();
for (const cat of cats) if (pool.filter(c => c.faction === cat).length >= 6)
add(`mono-${cat}`, curveDeck(pool, size, copies, c => c.faction === cat), ['arch:mono-faction']);
const kws = [...new Set(pool.map(c => c.keyword).filter((k): k is string => !!k))].sort();
for (const kw of kws) if (pool.filter(c => c.keyword === kw).length >= 3)
add(`kw-${kw}`, curveDeck(pool, size, copies, c => c.keyword === kw), ['arch:keyword']);
@@ -171,31 +184,53 @@ export function makeGameEngine(opts: GameEngineOptions = {}): EngineAdapter<Matc
decks(id) {
if (!deckCache.has(id)) {
const d = defaultDecks(rules(id), opts.extraDecks, opts.pool);
for (const deck of d) for (const c of deck.payload as GameCard[]) meta.set(c.id, c);
for (const deck of d) for (const c of deck.payload as GameCard[]) meta.set(baseId(c.cardId), c);
deckCache.set(id, d);
}
return deckCache.get(id)!;
},
createMatch: (id, decks, seed) => createMatch(rules(id), [decks[0].payload as GameCard[], decks[1].payload as GameCard[]], `sim-${seed}`),
// Simultaneous turns: seat 0 stages first, then seat 1. Bots never read the other side's staged plays
// Simultaneous turns: seat 0 queues first, then seat 1. Bots never read the other side's queued plays
// (evaluateFor hides them; lookahead determinizes), so the order leaks nothing.
toAct: s => (s.players[0].ended ? 1 : 0),
// Unstage is excluded: it never helps a bot and lets random bots loop.
legalActions: (s, seat) => legalActions(s, seat).filter(a => a.type !== 'unstage'),
toAct: s => (s.players[0].committed ? 1 : 0),
// Undo/timeout/concede are excluded: they never help a bot and let random bots loop.
legalActions: (s, seat) => legalActions(s, seat).filter(a => a.type === 'play' || a.type === 'commit'),
applyAction,
isOver,
result: s => { const r = result(s); return { winner: r?.winner ?? null, reason: r?.reason }; },
turn: s => s.turn,
evaluate: evaluateFor,
cardsOf: (_s, a) => (a.type === 'play' ? [a.cardId.replace(/:\d+$/, '')] : []),
cardsOf: (_s, a) => (a.type === 'play' ? [baseId(a.cardId)] : []),
actionKind: a => a.type,
cardMeta(id) {
const c = meta.get(id) ?? meta.get(`${id}:0`);
const c = meta.get(id);
if (!c) return undefined;
return { label: c.name, cost: c.cost, power: c.power, powerPerCost: c.power / Math.max(1, c.cost), keyword: c.keyword ?? 'vanilla', category: c.category };
return { label: c.title, cost: c.cost, power: c.power, powerPerCost: c.power / Math.max(1, c.cost), keyword: c.keyword ?? 'vanilla', faction: c.faction, home: c.home };
},
firstSeat: s => s.firstToReveal,
actionPower: (s, a) => (a.type === 'play' ? s.players[a.player].hand.find(c => c.id === a.cardId)?.power ?? 0 : -1),
firstSeat: s => s.initiative,
keywordValues(final, decks) {
const out: { seat: Seat; keyword: string; power: number; flip: number; cards: number }[] = [];
const r0 = final.result;
if (!r0) return out;
const payload = [decks[0].payload as GameCard[], decks[1].payload as GameCard[]] as const;
const score = (w: PlayerId | null, seat: Seat) => (w === null ? 0 : w === seat ? 1 : -1);
for (const seat of [0, 1] as Seat[]) {
const played = new Set(final.actions.filter(a => a.type === 'play' && a.player === seat).map(a => (a as { cardId: string }).cardId));
const byKw = new Map<string, number>();
for (const c of payload[seat]) if (c.keyword && played.has(c.cardId)) byKw.set(c.keyword, (byKw.get(c.keyword) ?? 0) + 1);
for (const [kw, cards] of byKw) {
const stripped = payload[seat].map(c => (c.keyword === kw ? { ...c, keyword: null } : c));
const decks2 = (seat === 0 ? [stripped, payload[1]] : [payload[0], stripped]) as [GameCard[], GameCard[]];
try {
const r1 = replay(final.ruleset, decks2, final.seed, final.actions).result;
if (!r1) continue;
out.push({ seat, keyword: kw, cards, power: r0.totalPower[seat] - r1.totalPower[seat], flip: score(r0.winner, seat) > score(r1.winner, seat) ? 1 : 0 });
} catch { /* replay diverged (e.g. a keyword changed legality); skip */ }
}
}
return out;
},
actionPower: (s, a) => (a.type === 'play' ? s.players[a.player].hand.find(c => c.cardId === a.cardId)?.power ?? 0 : -1),
determinize,
};
}
+7
View File
@@ -44,6 +44,12 @@ export interface EngineAdapter<S = unknown, A = unknown> {
cardMeta?(cardId: string): Record<string, number | string | string[]> | undefined;
/** Printed power an action puts on the board (-1 for non-plays). Used by the naive maxpower bot. */
actionPower?(state: S, action: A): number;
/**
* Counterfactual keyword value for a finished match: for each (seat, keyword) on cards that seat played,
* replay the same actions with that keyword removed from that seat's cards. `power` = own final total power
* lost, `flip` = 1 if the result got worse for that seat (win->draw/loss, draw->loss), `cards` = cards played.
*/
keywordValues?(final: S, decks: [DeckSpec, DeckSpec]): { seat: Seat; keyword: string; power: number; flip: number; cards: number }[];
/** Seat with the first-player advantage (defaults to toAct at match start). */
firstSeat?(state: S): Seat;
/** Resample information hidden from `seat` (opponent hand/staged plays, deck order) before searching. */
@@ -70,6 +76,7 @@ export interface MatchRecord {
/** Turn from which the eventual winner led on eval until the end (null for draws). */
decidedTurn: number | null;
cardsPlayed: [string[], string[]];
keywordValues?: { seat: Seat; keyword: string; power: number; flip: number; cards: number }[];
actionKinds: Record<string, number>;
ms: number;
}
+2 -2
View File
@@ -33,10 +33,10 @@ test('stats helpers', () => {
test('bots only return legal actions on shared/game', { skip: !existsSync(new URL('../shared/game/index.ts', import.meta.url)) }, async () => {
const { gameEngine } = await import('../shared/bots/game-adapter.ts');
const engine = gameEngine as EngineAdapter<any, any>;
const decks = engine.decks('base');
const decks = engine.decks(engine.rulesets()[0]);
for (const name of ['random', 'greedy', 'lookahead-lite']) {
const bot = makeBot(name, engine);
let s = engine.createMatch('base', [decks[0], decks[decks.length - 1]], 7);
let s = engine.createMatch(engine.rulesets()[0], [decks[0], decks[decks.length - 1]], 7);
const rng = mulberry32(1);
let steps = 0;
while (!engine.isOver(s) && steps++ < 500) {