H1: make bench, render budgets, and a third of the figure draw calls
make bench (tools/bench.sh, tools/bench_report.gd) measures the server's p50/p99/max tick with 24 bots on three seeds over a whole match, and the client's draw calls, primitives, objects, nodes and memory at six moments of a match rendered in the capture container. It fails on anything over the budgets now recorded in docs/ARCHITECTURE.md: p99 tick 12 ms, 2,000 draw calls and 2,000,000 primitives per frame (all passes), 1 GiB video and 512 MiB static memory. Every capture now reports its frame's render counts in a "render" field. The first run put the barge, with 24 figures together, at 2,197 draw calls. Each figure part had one surface per colour (about 43 per figure, drawn again for shadows). FigureRig now merges a part's surfaces into one per finish (coat, gilt, plain) and carries the Palette colours as vertex colours, with shared materials. The barge is down to 1,207 draw calls, with the same triangles and the same look. Frame rates on a real GPU are not measured here; that stays with R6. Closes #54 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,130 @@
|
||||
## H1: reads what tools/bench.sh measured, checks it against the performance budgets in
|
||||
## docs/ARCHITECTURE.md, and writes report.txt and report.json beside the logs.
|
||||
##
|
||||
## tools/godot.sh --script res://tools/bench_report.gd -- --dir=res://build/bench [--client=0]
|
||||
##
|
||||
## Exits 0 when everything is within budget, 1 when something isn't or nothing was
|
||||
## measured, 2 on a bad command line.
|
||||
extends SceneTree
|
||||
|
||||
## Server: the 99th percentile simulation tick, with 24 players (ServerLoop's budget).
|
||||
const SERVER_P99_USEC: int = ServerLoop.TICK_BUDGET_USEC
|
||||
## Client, per frame, every pass counted (shadows included). Set for a mid-range desktop
|
||||
## GPU from 2018 on, with room left for effects and the HUD.
|
||||
const CLIENT_DRAW_CALLS: int = 2000
|
||||
const CLIENT_PRIMITIVES: int = 2_000_000
|
||||
## Video memory the renderer reports using: fits a 2 GB card with room to spare.
|
||||
const CLIENT_VIDEO_MEM_BYTES: int = 1024 * 1024 * 1024
|
||||
## The engine's own static memory on the CPU side.
|
||||
const CLIENT_STATIC_MEM_BYTES: int = 512 * 1024 * 1024
|
||||
|
||||
var _failures: Array[String] = []
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
var dir: String = ""
|
||||
var want_client: bool = true
|
||||
for arg: String in OS.get_cmdline_user_args():
|
||||
if arg.begins_with("--dir="):
|
||||
dir = arg.trim_prefix("--dir=")
|
||||
elif arg.begins_with("--client="):
|
||||
want_client = arg.trim_prefix("--client=") != "0"
|
||||
else:
|
||||
printerr("bench_report: unknown argument '%s'" % arg)
|
||||
quit(2)
|
||||
return
|
||||
if dir.is_empty():
|
||||
printerr("bench_report: --dir is needed")
|
||||
quit(2)
|
||||
return
|
||||
var result: Dictionary = check(_read(dir.path_join("server.jsonl")), _read(dir.path_join("client.jsonl")), want_client)
|
||||
var text: String = describe(result)
|
||||
for failure: String in _failures:
|
||||
text += "bench FAIL: %s\n" % failure
|
||||
if _failures.is_empty():
|
||||
text += "bench: ok (everything within budget)\n"
|
||||
print(text.strip_edges())
|
||||
_write(dir.path_join("report.txt"), text)
|
||||
result["failures"] = _failures
|
||||
_write(dir.path_join("report.json"), JSON.stringify(result, "\t") + "\n")
|
||||
quit(0 if _failures.is_empty() else 1)
|
||||
|
||||
|
||||
## Checks parsed server "stats" lines and client capture reports. Returns the
|
||||
## measurements; failures go in `_failures`.
|
||||
func check(server: Array[Dictionary], client: Array[Dictionary], want_client: bool) -> Dictionary:
|
||||
var result: Dictionary = {"server": [], "client": []}
|
||||
if server.is_empty():
|
||||
_fail("no server runs were measured")
|
||||
for stats: Dictionary in server:
|
||||
var row: Dictionary = {
|
||||
"seed": int(stats.get("seed", -1)), "ticks": int(stats.get("ticks", 0)),
|
||||
"p50_us": int(stats.get("p50_us", -1)), "p99_us": int(stats.get("p99_us", -1)),
|
||||
"max_us": int(stats.get("max_us", -1)), "over_budget": int(stats.get("over_budget", 0)),
|
||||
}
|
||||
(result["server"] as Array).append(row)
|
||||
if row["p99_us"] < 0 or row["p99_us"] > SERVER_P99_USEC:
|
||||
_fail("server seed %d: p99 tick %d us is over the %d us budget" % [row["seed"], row["p99_us"], SERVER_P99_USEC])
|
||||
if want_client and client.is_empty():
|
||||
_fail("no client frames were measured")
|
||||
for capture: Dictionary in client:
|
||||
var render: Dictionary = capture.get("render", {})
|
||||
var row: Dictionary = {"scene": String(capture.get("scene", "?")).get_file().get_basename()}
|
||||
for key: String in ["draw_calls", "primitives", "objects", "visible_draw_calls", "visible_primitives", "shadow_draw_calls", "video_mem_bytes", "static_mem_bytes", "nodes"]:
|
||||
row[key] = int(render.get(key, -1))
|
||||
(result["client"] as Array).append(row)
|
||||
_limit(row, "draw_calls", CLIENT_DRAW_CALLS)
|
||||
_limit(row, "primitives", CLIENT_PRIMITIVES)
|
||||
_limit(row, "video_mem_bytes", CLIENT_VIDEO_MEM_BYTES)
|
||||
_limit(row, "static_mem_bytes", CLIENT_STATIC_MEM_BYTES)
|
||||
result["budgets"] = {
|
||||
"server_p99_us": SERVER_P99_USEC, "draw_calls": CLIENT_DRAW_CALLS, "primitives": CLIENT_PRIMITIVES,
|
||||
"video_mem_bytes": CLIENT_VIDEO_MEM_BYTES, "static_mem_bytes": CLIENT_STATIC_MEM_BYTES,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
## The measurements as a plain table.
|
||||
func describe(result: Dictionary) -> String:
|
||||
var text: String = "Server (24 bots, unthrottled; budget p99 %d us)\n" % SERVER_P99_USEC
|
||||
text += " %-10s %7s %8s %8s %8s %6s\n" % ["seed", "ticks", "p50 us", "p99 us", "max us", "over"]
|
||||
for row: Dictionary in result["server"]:
|
||||
text += " %-10d %7d %8d %8d %8d %6d\n" % [row["seed"], row["ticks"], row["p50_us"], row["p99_us"], row["max_us"], row["over_budget"]]
|
||||
if not (result["client"] as Array).is_empty():
|
||||
text += "Client frame (software rendering: counts, not speed; budgets %d draw calls, %d primitives, %d MiB video, %d MiB static)\n" % [
|
||||
CLIENT_DRAW_CALLS, CLIENT_PRIMITIVES, CLIENT_VIDEO_MEM_BYTES >> 20, CLIENT_STATIC_MEM_BYTES >> 20]
|
||||
text += " %-14s %6s %6s %9s %8s %7s %8s %8s %6s\n" % ["frame", "draws", "shadow", "prims", "objects", "visible", "video MB", "static MB", "nodes"]
|
||||
for row: Dictionary in result["client"]:
|
||||
text += " %-14s %6d %6d %9d %8d %7d %8d %8d %6d\n" % [
|
||||
row["scene"], row["draw_calls"], row["shadow_draw_calls"], row["primitives"], row["objects"],
|
||||
row["visible_draw_calls"], row["video_mem_bytes"] >> 20, row["static_mem_bytes"] >> 20, row["nodes"]]
|
||||
return text
|
||||
|
||||
|
||||
func _limit(row: Dictionary, key: String, budget: int) -> void:
|
||||
var value: int = row[key]
|
||||
if value < 0:
|
||||
_fail("%s: no %s was reported" % [row["scene"], key])
|
||||
elif value > budget:
|
||||
_fail("%s: %s %d is over the %d budget" % [row["scene"], key, value, budget])
|
||||
|
||||
|
||||
func _fail(message: String) -> void:
|
||||
_failures.append(message)
|
||||
|
||||
|
||||
func _read(path: String) -> Array[Dictionary]:
|
||||
var lines: Array[Dictionary] = []
|
||||
if not FileAccess.file_exists(path):
|
||||
return lines
|
||||
for line: String in FileAccess.get_file_as_string(path).split("\n", false):
|
||||
var parsed: Variant = JSON.parse_string(line)
|
||||
if parsed is Dictionary:
|
||||
lines.append(parsed)
|
||||
return lines
|
||||
|
||||
|
||||
func _write(path: String, text: String) -> void:
|
||||
var file: FileAccess = FileAccess.open(path, FileAccess.WRITE)
|
||||
if file != null:
|
||||
file.store_string(text)
|
||||
Reference in New Issue
Block a user