## M6: measures whether bots that build do better. Plays bot-only matches in which ## every other bot gathers and builds and the rest don't (BotBatch's split_builders), ## and compares the two groups. ## ## tools/godot.sh --script res://tools/bot_builds.gd -- [--seeds=20] [--first=501] [--out=FILE] ## ## Prints one JSON line per match and a summary, and checks: of the fights builders ## begin with the materials for a wall, they place a piece in at least STOCKED_SHARE; ## they lose a smaller share of the fights they built in than of the rest; their mean ## place is no worse than the control group's; and every match's p99 tick is within ## the budget. Exits 1 if a check fails. ## ## How the two groups' fight losses and wins compare overall is printed but not judged: ## over 40 matches (2026-11-15) they came out level, so building helps a bot in the ## fights it builds in without, yet, making builders win more. See docs/BALANCE.md. extends SceneTree ## Fewer matches than this are reported but not judged. const MIN_MATCHES: int = 20 ## Builders must place a piece in at least this share of the fights they begin with ## enough for a wall. (Most fights are in the landing, before anyone has gathered, so ## the share of all fights is reported but not judged.) const STOCKED_SHARE: float = 0.35 func _init() -> void: var count: int = 40 var first: int = 501 var out_path: String = "" for arg: String in OS.get_cmdline_user_args(): if arg.begins_with("--seeds="): count = arg.trim_prefix("--seeds=").to_int() elif arg.begins_with("--first="): first = arg.trim_prefix("--first=").to_int() elif arg.begins_with("--out="): out_path = arg.trim_prefix("--out=") var out: FileAccess = FileAccess.open(out_path, FileAccess.WRITE) if out_path != "" else null var results: Array[Dictionary] = [] var worst_p99: int = 0 var over_budget: int = 0 var unfinished: int = 0 for seed_value: int in range(first, first + count): var result: Dictionary = BotBatch.run_match(seed_value, Tuning.SOLO_PLAYERS, -1, true) results.append(result) worst_p99 = maxi(worst_p99, int(result["timing"]["p99_us"])) over_budget += int(result["timing"]["over_budget"]) unfinished += 0 if result["finished"] and result["winner"] != 0 else 1 var line: String = JSON.stringify({ "seed": seed_value, "ticks": result["ticks"], "winner": result["winner"], "p99_us": result["timing"]["p99_us"], "building": result["building"], }) print(line) if out != null: out.store_line(line) out.flush() var total: Dictionary = BotBatch.summarise_building(results) print(JSON.stringify({"summary": total, "worst_p99_us": worst_p99, "over_budget": over_budget})) var build: Dictionary = total["build"] var control: Dictionary = total["control"] print("bot-builds: builders placed %d pieces (%d walls, %d ramps, %d breakwaters) and gathered %d" % [build["pieces"], build["walls"], build["ramps"], build["breakwaters"], build["harvested"]]) print("bot-builds: builders began %.1f%% of their fights with the materials for a wall, and built in %.1f%% of those" % [build["stocked_share"] * 100.0, build["stocked_share_built"] * 100.0]) print("bot-builds: builders built in %.1f%% of %d fights and lost %.1f%% of them (%.1f%% when they built, %.1f%% when not); control lost %.1f%% of %d" % [ build["fight_share_built"] * 100.0, build["fights"], build["fight_loss"] * 100.0, build["loss_when_built"] * 100.0, build["loss_when_not"] * 100.0, control["fight_loss"] * 100.0, control["fights"]]) print("bot-builds: mean place builders %.2f, control %.2f; wins builders %d, control %d; worst p99 tick %d us" % [ build["mean_place"], control["mean_place"], build["wins"], control["wins"], worst_p99]) var problems: Array[String] = [] if unfinished > 0: problems.append("%d of %d matches didn't finish with a winner" % [unfinished, count]) if worst_p99 > ServerLoop.TICK_BUDGET_USEC: problems.append("a match's p99 tick was %d us, over the %d us budget" % [worst_p99, ServerLoop.TICK_BUDGET_USEC]) if count < MIN_MATCHES: print("bot-builds: %d matches are too few to judge (needs %d); only completion and timing were checked" % [count, MIN_MATCHES]) else: if build["stocked_share_built"] < STOCKED_SHARE: problems.append("builders built in only %.1f%% of the fights they could afford to (needs %.0f%%)" % [build["stocked_share_built"] * 100.0, STOCKED_SHARE * 100.0]) if build["loss_when_built"] >= build["loss_when_not"]: problems.append("builders lost %.1f%% of the fights they built in, no fewer than the %.1f%% they didn't" % [build["loss_when_built"] * 100.0, build["loss_when_not"] * 100.0]) if build["mean_place"] > control["mean_place"]: problems.append("builders' mean place %.2f is worse than control's %.2f" % [build["mean_place"], control["mean_place"]]) for problem: String in problems: print("bot-builds FAIL: %s" % problem) if problems.is_empty(): print("bot-builds: ok (%d matches)" % count) quit(0 if problems.is_empty() else 1)