The soak's client memory growth (17 to 20 MB, about five islands, in some processes) came from SoakPlayer keeping a strong reference to the last world it played on. It let go only when its client next asked it for a command on the new world, so finished islands stayed alive across a match change. It now keeps only the island's seed. LiveCount counts live Simulations and SimGrounds, which are made on worker threads too, so it is thread-safe. The soak clients log the counts, and the soak report fails if, a minute into a match, any island is alive that isn't a client's world or one being made. That makes a leak name itself instead of showing up as bytes. Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
236 lines
10 KiB
GDScript
236 lines
10 KiB
GDScript
## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the
|
|
## server ran its matches cleanly and within its tick budget, every client joined and
|
|
## played every match with no unreadable packets, every client's copy of the world
|
|
## matched the server's wherever both took a hash, and neither the server nor the
|
|
## client processes grew in memory from one match to the next.
|
|
##
|
|
## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH
|
|
## --client-logs=PATH[,PATH...] --clients=N --matches=N
|
|
##
|
|
## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own
|
|
## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line.
|
|
extends SceneTree
|
|
|
|
## How much a process's memory may grow from the first match to the last before it
|
|
## counts as a leak. Each match makes a new world, so some wobble is normal.
|
|
const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024
|
|
## Hash comparisons each client must reach per match, so a run that compared nothing
|
|
## can't pass.
|
|
const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10
|
|
|
|
var _failures: Array[String] = []
|
|
|
|
|
|
func _initialize() -> void:
|
|
var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"}
|
|
for arg: String in OS.get_cmdline_user_args():
|
|
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
|
if parts.size() != 2 or not options.has(parts[0]):
|
|
printerr("soak_report: unknown argument '%s'" % arg)
|
|
quit(2)
|
|
return
|
|
options[parts[0]] = parts[1]
|
|
if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty():
|
|
printerr("soak_report: --server-log and --client-logs are needed")
|
|
quit(2)
|
|
return
|
|
var result: Dictionary = check(
|
|
_read(String(options["server-log"])),
|
|
_read_all(String(options["client-logs"]).split(",")),
|
|
int(options["clients"]), int(options["matches"]))
|
|
print(JSON.stringify(result))
|
|
for failure: String in _failures:
|
|
print("soak FAIL: " + failure)
|
|
quit(0 if _failures.is_empty() else 1)
|
|
|
|
|
|
## Checks a run from its parsed log lines. Returns the measurements; failures go in
|
|
## `_failures`.
|
|
func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary:
|
|
var result: Dictionary = {"event": "soak"}
|
|
# The server.
|
|
var stats: Dictionary = _first(server, "stats")
|
|
var stop: Dictionary = _first(server, "stop")
|
|
if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0:
|
|
_fail("the server didn't stop cleanly: %s" % JSON.stringify(stop))
|
|
var ended: int = _count(server, "results")
|
|
result["matches_ended"] = ended
|
|
if ended < matches:
|
|
_fail("the server ended %d matches, not %d" % [ended, matches])
|
|
var budget: int = int(stats.get("budget_us", 0))
|
|
result["tick_p99_us"] = stats.get("with_net_p99_us", -1)
|
|
result["tick_max_us"] = stats.get("with_net_max_us", -1)
|
|
result["sim_p99_us"] = stats.get("p99_us", -1)
|
|
result["over_budget_ticks"] = stats.get("over_budget", -1)
|
|
if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget:
|
|
_fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget])
|
|
result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server")
|
|
# The clients.
|
|
var reports: Array[Dictionary] = _all(clients, "soak_client")
|
|
result["clients"] = reports.size()
|
|
if reports.size() != client_count:
|
|
_fail("%d clients reported, not %d" % [reports.size(), client_count])
|
|
var down: int = 0
|
|
var up: int = 0
|
|
var played: Dictionary = {}
|
|
for record: Dictionary in _all(clients, "client_match"):
|
|
played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1
|
|
for report: Dictionary in reports:
|
|
var number: int = int(report["client"])
|
|
down += int(report["down_bytes"])
|
|
up += int(report["up_bytes"])
|
|
if not report["joined"]:
|
|
_fail("client %d never joined" % number)
|
|
if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN:
|
|
_fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)])
|
|
if int(report["bad_packets"]) != 0:
|
|
_fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]])
|
|
if int(played.get(number, 0)) < matches:
|
|
_fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches])
|
|
var processes: Array[Dictionary] = _all(clients, "soak_process")
|
|
var seconds: int = 0
|
|
var mispredictions: int = 0
|
|
var reconciles: int = 0
|
|
var snaps: int = 0
|
|
for record: Dictionary in _all(clients, "client_match"):
|
|
mispredictions += int(record["mispredictions"])
|
|
reconciles += int(record["reconciles"])
|
|
snaps += int(record["snaps"])
|
|
var lost: int = 0
|
|
var sent: int = 0
|
|
for process: Dictionary in processes:
|
|
seconds = maxi(seconds, int(process["seconds"]))
|
|
lost += int(process["up_lost"]) + int(process["down_lost"])
|
|
sent += int(process["up_packets"]) + int(process["down_packets"])
|
|
if int(process["errors"]) != 0:
|
|
_fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")])
|
|
result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0)))
|
|
result["client_processes"] = processes.size()
|
|
result["seconds"] = seconds
|
|
result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001)
|
|
if seconds > 0 and not reports.is_empty():
|
|
result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
|
result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
|
result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1)
|
|
result["reconciles"] = reconciles
|
|
result["mispredictions"] = mispredictions
|
|
result["prediction_snaps"] = snaps
|
|
result["client_mem_growth_bytes"] = _client_growth(clients)
|
|
# The world, compared hash by hash.
|
|
var server_hashes: Dictionary = {}
|
|
for record: Dictionary in _all(server, "world_hash"):
|
|
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
|
|
var compared: int = 0
|
|
var differ: int = 0
|
|
var per_client: Dictionary = {}
|
|
for record: Dictionary in _all(clients, "hash"):
|
|
var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"]))
|
|
if not server_hashes.has(key):
|
|
continue
|
|
compared += 1
|
|
per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1
|
|
if server_hashes[key] != String(record["hash"]):
|
|
differ += 1
|
|
if differ <= 5:
|
|
_fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y])
|
|
result["hashes_compared"] = compared
|
|
result["hashes_differing"] = differ
|
|
if differ > 5:
|
|
_fail("%d world hashes differed in all" % differ)
|
|
for report: Dictionary in reports:
|
|
var number: int = int(report["client"])
|
|
if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches:
|
|
_fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)])
|
|
return result
|
|
|
|
|
|
## How much the server's memory grew from the first `phase` entry it logged to the last.
|
|
func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int:
|
|
var samples: Array[int] = []
|
|
for record: Dictionary in _all(lines, event):
|
|
if String(record.get("to", "")) == phase and record.has("mem_bytes"):
|
|
samples.append(int(record["mem_bytes"]))
|
|
return _judge(samples, who)
|
|
|
|
|
|
## The same for each client process, from its "memory" lines.
|
|
func _client_growth(lines: Array[Dictionary]) -> int:
|
|
var by_process: Dictionary = {}
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == "memory":
|
|
var process: int = int(record.get("_process", 0))
|
|
if not by_process.has(process):
|
|
by_process[process] = [] as Array[int]
|
|
(by_process[process] as Array[int]).append(int(record["mem_bytes"]))
|
|
var worst: int = 0
|
|
for process: int in by_process:
|
|
worst = maxi(worst, _judge(by_process[process], "client process %d" % process))
|
|
# A minute into each match, every island alive must be a client's world or one being
|
|
# made. Anything more is an island nothing should still hold (#39).
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) != "memory_late":
|
|
continue
|
|
var extra: int = int((record.get("worlds", {}) as Dictionary).get("extra_islands", 0))
|
|
if extra > 0:
|
|
_fail("client process %d still held %d islands beyond its clients' worlds in match %d" % [int(record.get("_process", 0)), extra, int(record.get("match", -1))])
|
|
return worst
|
|
|
|
|
|
func _judge(samples: Array[int], who: String) -> int:
|
|
if samples.size() < 2:
|
|
_fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()])
|
|
return 0
|
|
var growth: int = samples[samples.size() - 1] - samples[0]
|
|
if growth > LEAK_ALLOWANCE_BYTES:
|
|
_fail("%s grew by %d bytes from the first match to the last" % [who, growth])
|
|
return growth
|
|
|
|
|
|
func _fail(message: String) -> void:
|
|
_failures.append(message)
|
|
|
|
|
|
func _read_all(paths: PackedStringArray) -> Array[Dictionary]:
|
|
var lines: Array[Dictionary] = []
|
|
for index: int in paths.size():
|
|
for record: Dictionary in _read(paths[index]):
|
|
record["_process"] = index
|
|
lines.append(record)
|
|
return lines
|
|
|
|
|
|
## The JSON object lines of a log; anything else in it is skipped.
|
|
func _read(path: String) -> Array[Dictionary]:
|
|
var lines: Array[Dictionary] = []
|
|
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
|
|
if file == null:
|
|
_fail("couldn't read %s" % path)
|
|
return lines
|
|
while not file.eof_reached():
|
|
var line: String = file.get_line()
|
|
if not line.begins_with("{"):
|
|
continue
|
|
var parsed: Variant = JSON.parse_string(line)
|
|
if parsed is Dictionary:
|
|
lines.append(parsed)
|
|
return lines
|
|
|
|
|
|
func _first(lines: Array[Dictionary], event: String) -> Dictionary:
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == event:
|
|
return record
|
|
return {}
|
|
|
|
|
|
func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
|
|
var found: Array[Dictionary] = []
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == event:
|
|
found.append(record)
|
|
return found
|
|
|
|
|
|
func _count(lines: Array[Dictionary], event: String) -> int:
|
|
return _all(lines, event).size()
|