Files
wikideck/scripts/sim/run.ts
T

152 lines
7.5 KiB
TypeScript

// Balance simulation runner.
// npx tsx scripts/sim/run.ts --engine game --ruleset v1 --matches 5000 --workers 6 --out sim-out
// Writes <out>/<ruleset>.jsonl (records), <out>/<ruleset>.stats.json and <out>/<ruleset>.md.
import { fork } from 'node:child_process';
import { fileURLToPath } from 'node:url';
import { mkdirSync, writeFileSync, createWriteStream, readFileSync } from 'node:fs';
import { join } from 'node:path';
import { mulberry32 } from '../../shared/bots/index.ts';
import type { DeckSpec, EngineAdapter, MatchRecord } from '../../shared/bots/types.ts';
import { playMatch } from './match.ts';
import { aggregate, toMarkdown } from './stats.ts';
export async function loadEngine(name: string): Promise<EngineAdapter<any, any>> {
if (name === 'toy') return (await import('./toy-engine.ts')).toyEngine;
if (name === 'game') {
const { makeGameEngine } = await import('../../shared/bots/game-adapter.ts');
const { DEFAULT_RULESET } = await import('../../shared/game/index.ts');
const file = process.env.SIM_RULESETS;
const rulesets: Record<string, any> = { base: DEFAULT_RULESET };
if (file) {
// { "<id>": { "extends": "base", ...partial Ruleset } } — shallow-merged per top-level key.
const variants = JSON.parse(readFileSync(file, 'utf8')) as Record<string, any>;
for (const [id, v] of Object.entries(variants)) {
const { extends: parent = 'base', ...patch } = v;
const base = rulesets[parent];
const merged: any = { ...base };
for (const [k, val] of Object.entries(patch)) merged[k] = val && typeof val === 'object' && !Array.isArray(val) ? { ...base[k], ...val } : val;
rulesets[id] = merged;
}
}
let pool;
const extraDecks = [];
const cp = await import('./card-pool.ts');
if (process.env.SIM_STARTERS) extraDecks.push(...cp.loadStarterDecks(process.env.SIM_STARTERS));
if (process.env.SIM_POOL) {
const { loadPool } = cp;
pool = loadPool(process.env.SIM_POOL, { keywords: process.env.SIM_KEYWORDS === 'none' ? 'none' : 'faction', costThresholds: process.env.SIM_COST_THRESHOLDS ? JSON.parse(process.env.SIM_COST_THRESHOLDS) : undefined });
}
if (pool && process.env.SIM_OWNED) extraDecks.push(...cp.ownedDecks(pool, Number(process.env.SIM_OWNED)));
return makeGameEngine({ rulesets, pool, extraDecks });
}
throw new Error(`unknown engine ${name}`);
}
/** Bot pairings and their share of the schedule. Symmetric pairs carry the balance stats. */
export const DEFAULT_MIX: [string, string, number][] = [
['greedy', 'greedy', 0.55],
['lookahead-lite', 'lookahead-lite', 0.12],
['random', 'random', 0.05],
['greedy', 'lookahead-lite', 0.1],
['maxpower', 'lookahead-lite', 0.08],
['maxpower', 'greedy', 0.05],
['random', 'greedy', 0.05],
];
export const MIXES: Record<string, [string, string, number][]> = {
default: DEFAULT_MIX,
// Fast sweep: balance from greedy mirrors, plus a little skill signal.
fast: [['greedy', 'greedy', 0.8], ['maxpower', 'greedy', 0.1], ['random', 'greedy', 0.1]],
};
interface Job { i: number; seed: number; decks: [number, number]; bots: [string, string] }
export function schedule(n: number, nDecks: number, baseSeed: number, mix = DEFAULT_MIX): Job[] {
const rng = mulberry32(baseSeed);
const total = mix.reduce((t, m) => t + m[2], 0);
const jobs: Job[] = [];
for (let i = 0; i < n; i++) {
let r = rng() * total, k = 0;
while (r > mix[k][2] && k < mix.length - 1) { r -= mix[k][2]; k++; }
const flip = rng() < 0.5; // randomise seats so asymmetric pairs measure skill, not seat
const bots: [string, string] = flip ? [mix[k][1], mix[k][0]] : [mix[k][0], mix[k][1]];
jobs.push({ i, seed: (baseSeed * 1000003 + i * 7919) >>> 0, decks: [Math.floor(rng() * nDecks), Math.floor(rng() * nDecks)], bots });
}
return jobs;
}
/** Decks for a ruleset, filtered by the SIM_DECKS regex (inherited by child processes). */
export function decksFor(engine: EngineAdapter<any, any>, ruleset: string): DeckSpec[] {
const re = process.env.SIM_DECKS ? new RegExp(process.env.SIM_DECKS) : null;
return engine.decks(ruleset).filter(d => !re || re.test(d.id));
}
function runJobs(engine: EngineAdapter<any, any>, ruleset: string, decks: DeckSpec[], jobs: Job[], onRecord: (r: MatchRecord | { error: string; job: Job }) => void) {
for (const j of jobs) {
try { onRecord(playMatch(engine, ruleset, [decks[j.decks[0]], decks[j.decks[1]]], j.bots, j.seed)); }
catch (e) { onRecord({ error: String((e as Error)?.stack ?? e).slice(0, 600), job: j }); }
}
}
function arg(name: string, def?: string): string | undefined {
const i = process.argv.indexOf(`--${name}`);
return i >= 0 ? process.argv[i + 1] : def;
}
async function main() {
const engineName = arg('engine', 'toy')!;
const engine = await loadEngine(engineName);
const rulesets = (arg('ruleset') ?? engine.rulesets().join(',')).split(',');
const n = Number(arg('matches', '1000'));
const workers = Number(arg('workers', '6'));
const seed = Number(arg('seed', '1'));
const out = arg('out', 'sim-out')!;
mkdirSync(out, { recursive: true });
const sections: string[] = [];
for (const ruleset of rulesets) {
const decks = decksFor(engine, ruleset);
const jobs = schedule(n, decks.length, seed, MIXES[arg('mix', 'default')!]);
const records: MatchRecord[] = [];
const errors: string[] = [];
const file = createWriteStream(join(out, `${ruleset}.jsonl`));
const t0 = Date.now();
const handle = (r: any) => {
if (r.error) { errors.push(r.error); return; }
records.push(r);
file.write(JSON.stringify({ ...r, evalTrace: undefined, cardsPlayed: undefined, cards: r.cardsPlayed.map((x: string[]) => x.length) }) + '\n');
if ((records.length + errors.length) % 500 === 0) process.stderr.write(`[${ruleset}] ${records.length + errors.length}/${n} ${((Date.now() - t0) / 1000).toFixed(0)}s\n`);
};
if (workers <= 1) runJobs(engine, ruleset, decks, jobs, handle);
else {
const chunks: Job[][] = Array.from({ length: workers }, () => []);
jobs.forEach((j, i) => chunks[i % workers].push(j));
await Promise.all(chunks.map(chunk => new Promise<void>((resolve, reject) => {
const w = fork(fileURLToPath(import.meta.url), ['--child'], { execArgv: ['--import', 'tsx'], serialization: 'advanced' });
w.on('message', (m: any) => (m === 'done' ? w.kill() : handle(m)));
w.on('error', reject);
w.on('exit', () => resolve());
w.send({ engineName, ruleset, jobs: chunk });
})));
}
file.end();
const stats = aggregate(records, errors.length, engine);
writeFileSync(join(out, `${ruleset}.stats.json`), JSON.stringify(stats, null, 1));
if (errors.length) writeFileSync(join(out, `${ruleset}.errors.txt`), errors.slice(0, 50).join('\n----\n'));
const md = toMarkdown(stats);
writeFileSync(join(out, `${ruleset}.md`), md);
sections.push(md);
process.stderr.write(`[${ruleset}] done: ${records.length} matches, ${errors.length} errors, ${((Date.now() - t0) / 1000).toFixed(1)}s\n`);
}
process.stdout.write(sections.join('\n'));
}
const isEntry = !!process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
if (isEntry && process.argv.includes('--child')) {
process.once('message', async (msg: any) => {
const { engineName, ruleset, jobs } = msg as { engineName: string; ruleset: string; jobs: Job[] };
const engine = await loadEngine(engineName);
runJobs(engine, ruleset, decksFor(engine, ruleset), jobs, r => process.send!(r));
process.send!('done');
});
} else if (isEntry) main().catch(e => { console.error(e); process.exit(1); });