## H1: reads what tools/bench.sh measured, checks it against the performance budgets in ## docs/ARCHITECTURE.md, and writes report.txt and report.json beside the logs. ## ## tools/godot.sh --script res://tools/bench_report.gd -- --dir=res://build/bench [--client=0] ## ## Exits 0 when everything is within budget, 1 when something isn't or nothing was ## measured, 2 on a bad command line. extends SceneTree ## Server: the 99th percentile simulation tick, with 24 players (ServerLoop's budget). const SERVER_P99_USEC: int = ServerLoop.TICK_BUDGET_USEC ## Client, per frame, every pass counted (shadows included). Set for a mid-range desktop ## GPU from 2018 on, with room left for effects and the HUD. const CLIENT_DRAW_CALLS: int = 2000 const CLIENT_PRIMITIVES: int = 2_000_000 ## Video memory the renderer reports using: fits a 2 GB card with room to spare. const CLIENT_VIDEO_MEM_BYTES: int = 1024 * 1024 * 1024 ## The engine's own static memory on the CPU side. const CLIENT_STATIC_MEM_BYTES: int = 512 * 1024 * 1024 var _failures: Array[String] = [] func _initialize() -> void: var dir: String = "" var want_client: bool = true for arg: String in OS.get_cmdline_user_args(): if arg.begins_with("--dir="): dir = arg.trim_prefix("--dir=") elif arg.begins_with("--client="): want_client = arg.trim_prefix("--client=") != "0" else: printerr("bench_report: unknown argument '%s'" % arg) quit(2) return if dir.is_empty(): printerr("bench_report: --dir is needed") quit(2) return var result: Dictionary = check(_read(dir.path_join("server.jsonl")), _read(dir.path_join("client.jsonl")), want_client) var text: String = describe(result) for failure: String in _failures: text += "bench FAIL: %s\n" % failure if _failures.is_empty(): text += "bench: ok (everything within budget)\n" print(text.strip_edges()) _write(dir.path_join("report.txt"), text) result["failures"] = _failures _write(dir.path_join("report.json"), JSON.stringify(result, "\t") + "\n") quit(0 if _failures.is_empty() else 1) ## Checks parsed server "stats" lines and client capture reports. Returns the ## measurements; failures go in `_failures`. func check(server: Array[Dictionary], client: Array[Dictionary], want_client: bool) -> Dictionary: var result: Dictionary = {"server": [], "client": []} if server.is_empty(): _fail("no server runs were measured") for stats: Dictionary in server: var row: Dictionary = { "seed": int(stats.get("seed", -1)), "ticks": int(stats.get("ticks", 0)), "p50_us": int(stats.get("p50_us", -1)), "p99_us": int(stats.get("p99_us", -1)), "max_us": int(stats.get("max_us", -1)), "over_budget": int(stats.get("over_budget", 0)), } (result["server"] as Array).append(row) if row["p99_us"] < 0 or row["p99_us"] > SERVER_P99_USEC: _fail("server seed %d: p99 tick %d us is over the %d us budget" % [row["seed"], row["p99_us"], SERVER_P99_USEC]) if want_client and client.is_empty(): _fail("no client frames were measured") for capture: Dictionary in client: var render: Dictionary = capture.get("render", {}) var row: Dictionary = {"scene": String(capture.get("scene", "?")).get_file().get_basename()} for key: String in ["draw_calls", "primitives", "objects", "visible_draw_calls", "visible_primitives", "shadow_draw_calls", "video_mem_bytes", "static_mem_bytes", "nodes"]: row[key] = int(render.get(key, -1)) (result["client"] as Array).append(row) _limit(row, "draw_calls", CLIENT_DRAW_CALLS) _limit(row, "primitives", CLIENT_PRIMITIVES) _limit(row, "video_mem_bytes", CLIENT_VIDEO_MEM_BYTES) _limit(row, "static_mem_bytes", CLIENT_STATIC_MEM_BYTES) result["budgets"] = { "server_p99_us": SERVER_P99_USEC, "draw_calls": CLIENT_DRAW_CALLS, "primitives": CLIENT_PRIMITIVES, "video_mem_bytes": CLIENT_VIDEO_MEM_BYTES, "static_mem_bytes": CLIENT_STATIC_MEM_BYTES, } return result ## The measurements as a plain table. func describe(result: Dictionary) -> String: var text: String = "Server (24 bots, unthrottled; budget p99 %d us)\n" % SERVER_P99_USEC text += " %-10s %7s %8s %8s %8s %6s\n" % ["seed", "ticks", "p50 us", "p99 us", "max us", "over"] for row: Dictionary in result["server"]: text += " %-10d %7d %8d %8d %8d %6d\n" % [row["seed"], row["ticks"], row["p50_us"], row["p99_us"], row["max_us"], row["over_budget"]] if not (result["client"] as Array).is_empty(): text += "Client frame (software rendering: counts, not speed; budgets %d draw calls, %d primitives, %d MiB video, %d MiB static)\n" % [ CLIENT_DRAW_CALLS, CLIENT_PRIMITIVES, CLIENT_VIDEO_MEM_BYTES >> 20, CLIENT_STATIC_MEM_BYTES >> 20] text += " %-14s %6s %6s %9s %8s %7s %8s %8s %6s\n" % ["frame", "draws", "shadow", "prims", "objects", "visible", "video MB", "static MB", "nodes"] for row: Dictionary in result["client"]: text += " %-14s %6d %6d %9d %8d %7d %8d %8d %6d\n" % [ row["scene"], row["draw_calls"], row["shadow_draw_calls"], row["primitives"], row["objects"], row["visible_draw_calls"], row["video_mem_bytes"] >> 20, row["static_mem_bytes"] >> 20, row["nodes"]] return text func _limit(row: Dictionary, key: String, budget: int) -> void: var value: int = row[key] if value < 0: _fail("%s: no %s was reported" % [row["scene"], key]) elif value > budget: _fail("%s: %s %d is over the %d budget" % [row["scene"], key, value, budget]) func _fail(message: String) -> void: _failures.append(message) func _read(path: String) -> Array[Dictionary]: var lines: Array[Dictionary] = [] if not FileAccess.file_exists(path): return lines for line: String in FileAccess.get_file_as_string(path).split("\n", false): var parsed: Variant = JSON.parse_string(line) if parsed is Dictionary: lines.append(parsed) return lines func _write(path: String, text: String) -> void: var file: FileAccess = FileAccess.open(path, FileAccess.WRITE) if file != null: file.store_string(text)