157 lines
8.1 KiB
TypeScript
157 lines
8.1 KiB
TypeScript
// Balance simulation runner.
|
|
// npx tsx scripts/sim/run.ts --engine game --ruleset v1 --matches 5000 --workers 6 --out sim-out
|
|
// Writes <out>/<ruleset>.jsonl (records), <out>/<ruleset>.stats.json and <out>/<ruleset>.md.
|
|
import { fork } from 'node:child_process';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { mkdirSync, writeFileSync, createWriteStream, readFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { mulberry32 } from '../../shared/bots/index.ts';
|
|
import type { DeckSpec, EngineAdapter, MatchRecord } from '../../shared/bots/types.ts';
|
|
import { playMatch } from './match.ts';
|
|
import { aggregate, toMarkdown } from './stats.ts';
|
|
|
|
export async function loadEngine(name: string): Promise<EngineAdapter<any, any>> {
|
|
if (name === 'toy') return (await import('./toy-engine.ts')).toyEngine;
|
|
if (name === 'game') {
|
|
const { makeGameEngine } = await import('../../shared/bots/game-adapter.ts');
|
|
const game = await import('../../shared/game/index.ts') as Record<string, any>;
|
|
const { DEFAULT_RULESET } = game;
|
|
const file = process.env.SIM_RULESETS;
|
|
const rulesets: Record<string, any> = { base: DEFAULT_RULESET };
|
|
if (game.RULESET_V1_1) rulesets['v1.1'] = game.RULESET_V1_1; // engine-provided v1.1 (DESIGN #32)
|
|
if (file) {
|
|
// { "<id>": { "extends": "base", ...partial Ruleset } } — shallow-merged per top-level key.
|
|
const variants = JSON.parse(readFileSync(file, 'utf8')) as Record<string, any>;
|
|
for (const [id, v] of Object.entries(variants)) {
|
|
const { extends: parent = 'base', ...patch } = v;
|
|
const base = rulesets[parent];
|
|
const merged: any = { ...base };
|
|
for (const [k, val] of Object.entries(patch)) merged[k] = val && typeof val === 'object' && !Array.isArray(val) ? { ...base[k], ...val } : val;
|
|
rulesets[id] = merged;
|
|
}
|
|
}
|
|
let pool;
|
|
const extraDecks = [];
|
|
const cp = await import('./card-pool.ts');
|
|
if (process.env.SIM_STARTERS) extraDecks.push(...cp.loadStarterDecks(process.env.SIM_STARTERS));
|
|
if (process.env.SIM_POOL) {
|
|
const { loadPool } = cp;
|
|
pool = loadPool(process.env.SIM_POOL, { keywords: process.env.SIM_KEYWORDS === 'none' ? 'none' : 'faction', costThresholds: process.env.SIM_COST_THRESHOLDS ? JSON.parse(process.env.SIM_COST_THRESHOLDS) : undefined, cultureKeyword: process.env.SIM_CULTURE_KW === 'home' ? 'home' : 'faction' });
|
|
}
|
|
if (pool && process.env.SIM_OWNED) extraDecks.push(...cp.ownedDecks(pool, Number(process.env.SIM_OWNED)));
|
|
return makeGameEngine({ rulesets, pool, extraDecks });
|
|
}
|
|
throw new Error(`unknown engine ${name}`);
|
|
}
|
|
|
|
/** Bot pairings and their share of the schedule. Symmetric pairs carry the balance stats. */
|
|
export const DEFAULT_MIX: [string, string, number][] = [
|
|
['greedy', 'greedy', 0.55],
|
|
['lookahead-lite', 'lookahead-lite', 0.12],
|
|
['random', 'random', 0.05],
|
|
['greedy', 'lookahead-lite', 0.1],
|
|
['maxpower', 'lookahead-lite', 0.08],
|
|
['maxpower', 'greedy', 0.05],
|
|
['random', 'greedy', 0.05],
|
|
];
|
|
|
|
export const MIXES: Record<string, [string, string, number][]> = {
|
|
default: DEFAULT_MIX,
|
|
// Report mix: balance from greedy mirrors, lighter lookahead share (cheaper on CPU and memory).
|
|
report: [['greedy', 'greedy', 0.7], ['lookahead-lite', 'lookahead-lite', 0.04], ['greedy', 'lookahead-lite', 0.06],
|
|
['maxpower', 'greedy', 0.08], ['random', 'greedy', 0.06], ['random', 'random', 0.06]],
|
|
// Fast sweep: balance from greedy mirrors, plus a little skill signal.
|
|
fast: [['greedy', 'greedy', 0.8], ['maxpower', 'greedy', 0.1], ['random', 'greedy', 0.1]],
|
|
};
|
|
|
|
interface Job { i: number; seed: number; decks: [number, number]; bots: [string, string] }
|
|
|
|
export function schedule(n: number, nDecks: number, baseSeed: number, mix = DEFAULT_MIX): Job[] {
|
|
const rng = mulberry32(baseSeed);
|
|
const total = mix.reduce((t, m) => t + m[2], 0);
|
|
const jobs: Job[] = [];
|
|
for (let i = 0; i < n; i++) {
|
|
let r = rng() * total, k = 0;
|
|
while (r > mix[k][2] && k < mix.length - 1) { r -= mix[k][2]; k++; }
|
|
const flip = rng() < 0.5; // randomise seats so asymmetric pairs measure skill, not seat
|
|
const bots: [string, string] = flip ? [mix[k][1], mix[k][0]] : [mix[k][0], mix[k][1]];
|
|
jobs.push({ i, seed: (baseSeed * 1000003 + i * 7919) >>> 0, decks: [Math.floor(rng() * nDecks), Math.floor(rng() * nDecks)], bots });
|
|
}
|
|
return jobs;
|
|
}
|
|
|
|
/** Decks for a ruleset, filtered by the SIM_DECKS regex (inherited by child processes). */
|
|
export function decksFor(engine: EngineAdapter<any, any>, ruleset: string): DeckSpec[] {
|
|
const re = process.env.SIM_DECKS ? new RegExp(process.env.SIM_DECKS) : null;
|
|
return engine.decks(ruleset).filter(d => !re || re.test(d.id));
|
|
}
|
|
|
|
function runJobs(engine: EngineAdapter<any, any>, ruleset: string, decks: DeckSpec[], jobs: Job[], onRecord: (r: MatchRecord | { error: string; job: Job }) => void) {
|
|
for (const j of jobs) {
|
|
try { onRecord(playMatch(engine, ruleset, [decks[j.decks[0]], decks[j.decks[1]]], j.bots, j.seed)); }
|
|
catch (e) { onRecord({ error: String((e as Error)?.stack ?? e).slice(0, 600), job: j }); }
|
|
}
|
|
}
|
|
|
|
function arg(name: string, def?: string): string | undefined {
|
|
const i = process.argv.indexOf(`--${name}`);
|
|
return i >= 0 ? process.argv[i + 1] : def;
|
|
}
|
|
|
|
async function main() {
|
|
const engineName = arg('engine', 'toy')!;
|
|
const engine = await loadEngine(engineName);
|
|
const rulesets = (arg('ruleset') ?? engine.rulesets().join(',')).split(',');
|
|
const n = Number(arg('matches', '1000'));
|
|
const workers = Number(arg('workers', '6'));
|
|
const seed = Number(arg('seed', '1'));
|
|
const out = arg('out', 'sim-out')!;
|
|
mkdirSync(out, { recursive: true });
|
|
const sections: string[] = [];
|
|
for (const ruleset of rulesets) {
|
|
const decks = decksFor(engine, ruleset);
|
|
const jobs = schedule(n, decks.length, seed, MIXES[arg('mix', 'default')!]);
|
|
const records: MatchRecord[] = [];
|
|
const errors: string[] = [];
|
|
const file = createWriteStream(join(out, `${ruleset}.jsonl`));
|
|
const t0 = Date.now();
|
|
const handle = (r: any) => {
|
|
if (r.error) { errors.push(r.error); return; }
|
|
records.push(r);
|
|
file.write(JSON.stringify({ ...r, evalTrace: undefined, cardsPlayed: undefined, cards: r.cardsPlayed.map((x: string[]) => x.length) }) + '\n');
|
|
if ((records.length + errors.length) % 500 === 0) process.stderr.write(`[${ruleset}] ${records.length + errors.length}/${n} ${((Date.now() - t0) / 1000).toFixed(0)}s\n`);
|
|
};
|
|
if (workers <= 1) runJobs(engine, ruleset, decks, jobs, handle);
|
|
else {
|
|
const chunks: Job[][] = Array.from({ length: workers }, () => []);
|
|
jobs.forEach((j, i) => chunks[i % workers].push(j));
|
|
await Promise.all(chunks.map(chunk => new Promise<void>((resolve, reject) => {
|
|
const w = fork(fileURLToPath(import.meta.url), ['--child'], { execArgv: ['--import', 'tsx'], serialization: 'advanced' });
|
|
w.on('message', (m: any) => (m === 'done' ? w.kill() : handle(m)));
|
|
w.on('error', reject);
|
|
w.on('exit', () => resolve());
|
|
w.send({ engineName, ruleset, jobs: chunk });
|
|
})));
|
|
}
|
|
file.end();
|
|
const stats = aggregate(records, errors.length, engine);
|
|
writeFileSync(join(out, `${ruleset}.stats.json`), JSON.stringify(stats, null, 1));
|
|
if (errors.length) writeFileSync(join(out, `${ruleset}.errors.txt`), errors.slice(0, 50).join('\n----\n'));
|
|
const md = toMarkdown(stats);
|
|
writeFileSync(join(out, `${ruleset}.md`), md);
|
|
sections.push(md);
|
|
process.stderr.write(`[${ruleset}] done: ${records.length} matches, ${errors.length} errors, ${((Date.now() - t0) / 1000).toFixed(1)}s\n`);
|
|
}
|
|
process.stdout.write(sections.join('\n'));
|
|
}
|
|
|
|
const isEntry = !!process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
|
|
if (isEntry && process.argv.includes('--child')) {
|
|
process.once('message', async (msg: any) => {
|
|
const { engineName, ruleset, jobs } = msg as { engineName: string; ruleset: string; jobs: Job[] };
|
|
const engine = await loadEngine(engineName);
|
|
runJobs(engine, ruleset, decksFor(engine, ruleset), jobs, r => process.send!(r));
|
|
process.send!('done');
|
|
});
|
|
} else if (isEntry) main().catch(e => { console.error(e); process.exit(1); });
|