## Plays bot-only matches on a run of seeds and checks issue M5's acceptance: every ## match ends with a winner, under a tenth of deaths are drownings, and difficulty ## shows: easy bots win less than an even share and take the fewest eliminations. ## ## tools/godot.sh --script res://tools/bot_matches.gd -- [--seeds=20] [--first=1] ## ## Prints one JSON line per match and a summary. Exits 1 if a check fails. extends SceneTree ## The difficulty checks need a batch this big to mean anything. const DIFFICULTY_MIN_MATCHES: int = 20 func _init() -> void: var count: int = 20 var first: int = 1 for arg: String in OS.get_cmdline_user_args(): if arg.begins_with("--seeds="): count = arg.trim_prefix("--seeds=").to_int() elif arg.begins_with("--first="): first = arg.trim_prefix("--first=").to_int() var results: Array[Dictionary] = [] for seed_value: int in range(first, first + count): var result: Dictionary = BotBatch.run_match(seed_value) results.append(result) print(JSON.stringify(result)) var total: Dictionary = BotBatch.summarise(results) print(JSON.stringify({"summary": total})) var problems: Array[String] = [] if total["with_winner"] != count or total["finished"] != count: problems.append("%d of %d matches finished with a winner" % [total["with_winner"], count]) if total["drown_share"] >= 0.1: problems.append("%.1f%% of deaths were drownings" % (total["drown_share"] * 100.0)) # What is checked is what held on every 20-match batch measured (issue #76): easy # bots win less than an even share of matches and take the fewest eliminations. # Mean place doesn't separate the levels over 20 matches, and nothing measured in # whole matches reliably separates hard from normal; the duel test in # test/bots/test_bot_brain.gd covers hard against easy. See issue #56. var mean: Dictionary = total["mean_place"] var kills: Dictionary = total["eliminations"] var wins: Dictionary = total["wins"] var easy_wins: int = int(wins.get("easy", 0)) if count < DIFFICULTY_MIN_MATCHES: print("bot-matches: %d matches are too few to judge difficulty (needs %d); those checks were skipped" % [count, DIFFICULTY_MIN_MATCHES]) else: if easy_wins * Tuning.BOT_DIFFICULTIES.size() >= count: problems.append("easy bots won %d of %d matches, an even share or more: %s" % [easy_wins, count, wins]) if not (kills.get("easy", 0) < kills.get("normal", 0) and kills.get("easy", 0) < kills.get("hard", 0)): problems.append("easy bots didn't take the fewest eliminations: %s" % kills) for problem: String in problems: print("bot-matches FAIL: %s" % problem) if problems.is_empty(): print("bot-matches: ok (%d matches, all with a winner, %.1f%% drowned, wins %s, eliminations %s, mean place %s)" % [count, total["drown_share"] * 100.0, wins, kills, mean]) quit(0 if problems.is_empty() else 1)