Files
tidewall/tools/bot_matches.gd
T
claudeandClaude Opus 5.5 881f449d10 M9: Bot batch checks what holds between difficulties
`make bot-matches` failed on its default seeds because its difficulty checks
(easy places worst, hard takes the most eliminations) are noisier than the
differences they look for over 20 matches.

Measured on three 20-seed batches (seeds 1-20, 21-40, 101-120): easy bots won
0, 4 and 2 matches against an even share of 6.7, and took the fewest
eliminations every time. Mean place doesn't separate the levels, and hard is
not reliably ahead of normal by any whole-match measure.

The batch now checks that easy bots win less than an even share and take the
fewest eliminations, and skips the difficulty checks under 20 matches, where
they mean nothing. Hard against easy stays covered by the duel test.

Closes #76

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 08:26:02 -05:00

55 lines
2.8 KiB
GDScript

## Plays bot-only matches on a run of seeds and checks issue M5's acceptance: every
## match ends with a winner, under a tenth of deaths are drownings, and difficulty
## shows: easy bots win less than an even share and take the fewest eliminations.
##
## tools/godot.sh --script res://tools/bot_matches.gd -- [--seeds=20] [--first=1]
##
## Prints one JSON line per match and a summary. Exits 1 if a check fails.
extends SceneTree
## The difficulty checks need a batch this big to mean anything.
const DIFFICULTY_MIN_MATCHES: int = 20
func _init() -> void:
var count: int = 20
var first: int = 1
for arg: String in OS.get_cmdline_user_args():
if arg.begins_with("--seeds="):
count = arg.trim_prefix("--seeds=").to_int()
elif arg.begins_with("--first="):
first = arg.trim_prefix("--first=").to_int()
var results: Array[Dictionary] = []
for seed_value: int in range(first, first + count):
var result: Dictionary = BotBatch.run_match(seed_value)
results.append(result)
print(JSON.stringify(result))
var total: Dictionary = BotBatch.summarise(results)
print(JSON.stringify({"summary": total}))
var problems: Array[String] = []
if total["with_winner"] != count or total["finished"] != count:
problems.append("%d of %d matches finished with a winner" % [total["with_winner"], count])
if total["drown_share"] >= 0.1:
problems.append("%.1f%% of deaths were drownings" % (total["drown_share"] * 100.0))
# What is checked is what held on every 20-match batch measured (issue #76): easy
# bots win less than an even share of matches and take the fewest eliminations.
# Mean place doesn't separate the levels over 20 matches, and nothing measured in
# whole matches reliably separates hard from normal; the duel test in
# test/bots/test_bot_brain.gd covers hard against easy. See issue #56.
var mean: Dictionary = total["mean_place"]
var kills: Dictionary = total["eliminations"]
var wins: Dictionary = total["wins"]
var easy_wins: int = int(wins.get("easy", 0))
if count < DIFFICULTY_MIN_MATCHES:
print("bot-matches: %d matches are too few to judge difficulty (needs %d); those checks were skipped" % [count, DIFFICULTY_MIN_MATCHES])
else:
if easy_wins * Tuning.BOT_DIFFICULTIES.size() >= count:
problems.append("easy bots won %d of %d matches, an even share or more: %s" % [easy_wins, count, wins])
if not (kills.get("easy", 0) < kills.get("normal", 0) and kills.get("easy", 0) < kills.get("hard", 0)):
problems.append("easy bots didn't take the fewest eliminations: %s" % kills)
for problem: String in problems:
print("bot-matches FAIL: %s" % problem)
if problems.is_empty():
print("bot-matches: ok (%d matches, all with a winner, %.1f%% drowned, wins %s, eliminations %s, mean place %s)" % [count, total["drown_share"] * 100.0, wins, kills, mean])
quit(0 if problems.is_empty() else 1)