Files
wikideck/scripts/sim/run.ts
T
claudeandClaude Opus 5.5 9841622ac7 sim-balance: bots (random/greedy/lookahead), sim runner and stats
Bots are pure (state, player, rng) => action over an EngineAdapter;
shared/bots/game-adapter.ts adapts the shared/game API (hidden staged
plays, determinized lookahead). scripts/sim/run.ts runs N matches per
ruleset across child processes and writes per-ruleset markdown/JSON.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 22:54:53 -05:00

129 lines
6.3 KiB
TypeScript

// Balance simulation runner.
// npx tsx scripts/sim/run.ts --engine game --ruleset v1 --matches 5000 --workers 6 --out sim-out
// Writes <out>/<ruleset>.jsonl (records), <out>/<ruleset>.stats.json and <out>/<ruleset>.md.
import { fork } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { mkdirSync, writeFileSync, createWriteStream, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { mulberry32 } from '../../shared/bots/index.ts';
import type { DeckSpec, EngineAdapter, MatchRecord } from '../../shared/bots/types.ts';
import { playMatch } from './match.ts';
import { aggregate, toMarkdown } from './stats.ts';
export async function loadEngine(name: string): Promise<EngineAdapter<any, any>> {
if (name === 'toy') return (await import('./toy-engine.ts')).toyEngine;
if (name === 'game') {
const { makeGameEngine } = await import('../../shared/bots/game-adapter.ts');
const { DEFAULT_RULESET } = await import('../../shared/game/index.ts');
const file = process.env.SIM_RULESETS;
const rulesets: Record<string, any> = { base: DEFAULT_RULESET };
if (file) {
// { "<id>": { "extends": "base", ...partial Ruleset } } — shallow-merged per top-level key.
const variants = JSON.parse(readFileSync(file, 'utf8')) as Record<string, any>;
for (const [id, v] of Object.entries(variants)) {
const { extends: parent = 'base', ...patch } = v;
const base = rulesets[parent];
const merged: any = { ...base };
for (const [k, val] of Object.entries(patch)) merged[k] = val && typeof val === 'object' && !Array.isArray(val) ? { ...base[k], ...val } : val;
rulesets[id] = { ...merged, id: v.id ?? `sim-${id}`.replace(/[^a-z0-9-]/g, '-').slice(0, 48) };
}
}
return makeGameEngine({ rulesets });
}
throw new Error(`unknown engine ${name}`);
}
/** Bot pairings and their share of the schedule. Symmetric pairs carry the balance stats. */
export const DEFAULT_MIX: [string, string, number][] = [
['greedy', 'greedy', 0.55],
['lookahead-lite', 'lookahead-lite', 0.15],
['random', 'random', 0.1],
['greedy', 'lookahead-lite', 0.1],
['random', 'greedy', 0.1],
];
interface Job { i: number; seed: number; decks: [number, number]; bots: [string, string] }
export function schedule(n: number, nDecks: number, baseSeed: number, mix = DEFAULT_MIX): Job[] {
const rng = mulberry32(baseSeed);
const total = mix.reduce((t, m) => t + m[2], 0);
const jobs: Job[] = [];
for (let i = 0; i < n; i++) {
let r = rng() * total, k = 0;
while (r > mix[k][2] && k < mix.length - 1) { r -= mix[k][2]; k++; }
const flip = rng() < 0.5; // randomise seats so asymmetric pairs measure skill, not seat
const bots: [string, string] = flip ? [mix[k][1], mix[k][0]] : [mix[k][0], mix[k][1]];
jobs.push({ i, seed: (baseSeed * 1000003 + i * 7919) >>> 0, decks: [Math.floor(rng() * nDecks), Math.floor(rng() * nDecks)], bots });
}
return jobs;
}
function runJobs(engine: EngineAdapter<any, any>, ruleset: string, decks: DeckSpec[], jobs: Job[], onRecord: (r: MatchRecord | { error: string; job: Job }) => void) {
for (const j of jobs) {
try { onRecord(playMatch(engine, ruleset, [decks[j.decks[0]], decks[j.decks[1]]], j.bots, j.seed)); }
catch (e) { onRecord({ error: String((e as Error)?.stack ?? e).slice(0, 600), job: j }); }
}
}
function arg(name: string, def?: string): string | undefined {
const i = process.argv.indexOf(`--${name}`);
return i >= 0 ? process.argv[i + 1] : def;
}
async function main() {
const engineName = arg('engine', 'toy')!;
const engine = await loadEngine(engineName);
const rulesets = (arg('ruleset') ?? engine.rulesets().join(',')).split(',');
const n = Number(arg('matches', '1000'));
const workers = Number(arg('workers', '6'));
const seed = Number(arg('seed', '1'));
const out = arg('out', 'sim-out')!;
mkdirSync(out, { recursive: true });
const sections: string[] = [];
for (const ruleset of rulesets) {
const decks = engine.decks(ruleset);
const jobs = schedule(n, decks.length, seed);
const records: MatchRecord[] = [];
const errors: string[] = [];
const file = createWriteStream(join(out, `${ruleset}.jsonl`));
const t0 = Date.now();
const handle = (r: any) => {
if (r.error) { errors.push(r.error); return; }
records.push(r);
file.write(JSON.stringify({ ...r, evalTrace: undefined, cardsPlayed: undefined, cards: r.cardsPlayed.map((x: string[]) => x.length) }) + '\n');
if ((records.length + errors.length) % 500 === 0) process.stderr.write(`[${ruleset}] ${records.length + errors.length}/${n} ${((Date.now() - t0) / 1000).toFixed(0)}s\n`);
};
if (workers <= 1) runJobs(engine, ruleset, decks, jobs, handle);
else {
const chunks: Job[][] = Array.from({ length: workers }, () => []);
jobs.forEach((j, i) => chunks[i % workers].push(j));
await Promise.all(chunks.map(chunk => new Promise<void>((resolve, reject) => {
const w = fork(fileURLToPath(import.meta.url), ['--child'], { execArgv: ['--import', 'tsx'], serialization: 'advanced' });
w.on('message', (m: any) => (m === 'done' ? w.kill() : handle(m)));
w.on('error', reject);
w.on('exit', () => resolve());
w.send({ engineName, ruleset, jobs: chunk });
})));
}
file.end();
const stats = aggregate(records, errors.length, engine);
writeFileSync(join(out, `${ruleset}.stats.json`), JSON.stringify(stats, null, 1));
if (errors.length) writeFileSync(join(out, `${ruleset}.errors.txt`), errors.slice(0, 50).join('\n----\n'));
const md = toMarkdown(stats);
writeFileSync(join(out, `${ruleset}.md`), md);
sections.push(md);
process.stderr.write(`[${ruleset}] done: ${records.length} matches, ${errors.length} errors, ${((Date.now() - t0) / 1000).toFixed(1)}s\n`);
}
process.stdout.write(sections.join('\n'));
}
const isEntry = !!process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
if (isEntry && process.argv.includes('--child')) {
process.once('message', async (msg: any) => {
const { engineName, ruleset, jobs } = msg as { engineName: string; ruleset: string; jobs: Job[] };
const engine = await loadEngine(engineName);
runJobs(engine, ruleset, engine.decks(ruleset), jobs, r => process.send!(r));
process.send!('done');
});
} else if (isEntry) main().catch(e => { console.error(e); process.exit(1); });