Files
tidewall/tools/soak_report.gd
T
claudeandClaude Opus 5.5 880e001fbb N8: Stop the soak player holding old islands, and count live islands
The soak's client memory growth (17 to 20 MB, about five islands, in some
processes) came from SoakPlayer keeping a strong reference to the last world it
played on. It let go only when its client next asked it for a command on the
new world, so finished islands stayed alive across a match change. It now keeps
only the island's seed.

LiveCount counts live Simulations and SimGrounds, which are made on worker
threads too, so it is thread-safe. The soak clients log the counts, and the
soak report fails if, a minute into a match, any island is alive that isn't a
client's world or one being made. That makes a leak name itself instead of
showing up as bytes.

Refs #39

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 09:01:49 -05:00

236 lines
10 KiB
GDScript

## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the
## server ran its matches cleanly and within its tick budget, every client joined and
## played every match with no unreadable packets, every client's copy of the world
## matched the server's wherever both took a hash, and neither the server nor the
## client processes grew in memory from one match to the next.
##
## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH
## --client-logs=PATH[,PATH...] --clients=N --matches=N
##
## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own
## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line.
extends SceneTree
## How much a process's memory may grow from the first match to the last before it
## counts as a leak. Each match makes a new world, so some wobble is normal.
const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024
## Hash comparisons each client must reach per match, so a run that compared nothing
## can't pass.
const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10
var _failures: Array[String] = []
func _initialize() -> void:
var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"}
for arg: String in OS.get_cmdline_user_args():
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
if parts.size() != 2 or not options.has(parts[0]):
printerr("soak_report: unknown argument '%s'" % arg)
quit(2)
return
options[parts[0]] = parts[1]
if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty():
printerr("soak_report: --server-log and --client-logs are needed")
quit(2)
return
var result: Dictionary = check(
_read(String(options["server-log"])),
_read_all(String(options["client-logs"]).split(",")),
int(options["clients"]), int(options["matches"]))
print(JSON.stringify(result))
for failure: String in _failures:
print("soak FAIL: " + failure)
quit(0 if _failures.is_empty() else 1)
## Checks a run from its parsed log lines. Returns the measurements; failures go in
## `_failures`.
func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary:
var result: Dictionary = {"event": "soak"}
# The server.
var stats: Dictionary = _first(server, "stats")
var stop: Dictionary = _first(server, "stop")
if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0:
_fail("the server didn't stop cleanly: %s" % JSON.stringify(stop))
var ended: int = _count(server, "results")
result["matches_ended"] = ended
if ended < matches:
_fail("the server ended %d matches, not %d" % [ended, matches])
var budget: int = int(stats.get("budget_us", 0))
result["tick_p99_us"] = stats.get("with_net_p99_us", -1)
result["tick_max_us"] = stats.get("with_net_max_us", -1)
result["sim_p99_us"] = stats.get("p99_us", -1)
result["over_budget_ticks"] = stats.get("over_budget", -1)
if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget:
_fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget])
result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server")
# The clients.
var reports: Array[Dictionary] = _all(clients, "soak_client")
result["clients"] = reports.size()
if reports.size() != client_count:
_fail("%d clients reported, not %d" % [reports.size(), client_count])
var down: int = 0
var up: int = 0
var played: Dictionary = {}
for record: Dictionary in _all(clients, "client_match"):
played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1
for report: Dictionary in reports:
var number: int = int(report["client"])
down += int(report["down_bytes"])
up += int(report["up_bytes"])
if not report["joined"]:
_fail("client %d never joined" % number)
if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN:
_fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)])
if int(report["bad_packets"]) != 0:
_fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]])
if int(played.get(number, 0)) < matches:
_fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches])
var processes: Array[Dictionary] = _all(clients, "soak_process")
var seconds: int = 0
var mispredictions: int = 0
var reconciles: int = 0
var snaps: int = 0
for record: Dictionary in _all(clients, "client_match"):
mispredictions += int(record["mispredictions"])
reconciles += int(record["reconciles"])
snaps += int(record["snaps"])
var lost: int = 0
var sent: int = 0
for process: Dictionary in processes:
seconds = maxi(seconds, int(process["seconds"]))
lost += int(process["up_lost"]) + int(process["down_lost"])
sent += int(process["up_packets"]) + int(process["down_packets"])
if int(process["errors"]) != 0:
_fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")])
result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0)))
result["client_processes"] = processes.size()
result["seconds"] = seconds
result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001)
if seconds > 0 and not reports.is_empty():
result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1)
result["reconciles"] = reconciles
result["mispredictions"] = mispredictions
result["prediction_snaps"] = snaps
result["client_mem_growth_bytes"] = _client_growth(clients)
# The world, compared hash by hash.
var server_hashes: Dictionary = {}
for record: Dictionary in _all(server, "world_hash"):
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
var compared: int = 0
var differ: int = 0
var per_client: Dictionary = {}
for record: Dictionary in _all(clients, "hash"):
var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"]))
if not server_hashes.has(key):
continue
compared += 1
per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1
if server_hashes[key] != String(record["hash"]):
differ += 1
if differ <= 5:
_fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y])
result["hashes_compared"] = compared
result["hashes_differing"] = differ
if differ > 5:
_fail("%d world hashes differed in all" % differ)
for report: Dictionary in reports:
var number: int = int(report["client"])
if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches:
_fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)])
return result
## How much the server's memory grew from the first `phase` entry it logged to the last.
func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int:
var samples: Array[int] = []
for record: Dictionary in _all(lines, event):
if String(record.get("to", "")) == phase and record.has("mem_bytes"):
samples.append(int(record["mem_bytes"]))
return _judge(samples, who)
## The same for each client process, from its "memory" lines.
func _client_growth(lines: Array[Dictionary]) -> int:
var by_process: Dictionary = {}
for record: Dictionary in lines:
if String(record.get("event", "")) == "memory":
var process: int = int(record.get("_process", 0))
if not by_process.has(process):
by_process[process] = [] as Array[int]
(by_process[process] as Array[int]).append(int(record["mem_bytes"]))
var worst: int = 0
for process: int in by_process:
worst = maxi(worst, _judge(by_process[process], "client process %d" % process))
# A minute into each match, every island alive must be a client's world or one being
# made. Anything more is an island nothing should still hold (#39).
for record: Dictionary in lines:
if String(record.get("event", "")) != "memory_late":
continue
var extra: int = int((record.get("worlds", {}) as Dictionary).get("extra_islands", 0))
if extra > 0:
_fail("client process %d still held %d islands beyond its clients' worlds in match %d" % [int(record.get("_process", 0)), extra, int(record.get("match", -1))])
return worst
func _judge(samples: Array[int], who: String) -> int:
if samples.size() < 2:
_fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()])
return 0
var growth: int = samples[samples.size() - 1] - samples[0]
if growth > LEAK_ALLOWANCE_BYTES:
_fail("%s grew by %d bytes from the first match to the last" % [who, growth])
return growth
func _fail(message: String) -> void:
_failures.append(message)
func _read_all(paths: PackedStringArray) -> Array[Dictionary]:
var lines: Array[Dictionary] = []
for index: int in paths.size():
for record: Dictionary in _read(paths[index]):
record["_process"] = index
lines.append(record)
return lines
## The JSON object lines of a log; anything else in it is skipped.
func _read(path: String) -> Array[Dictionary]:
var lines: Array[Dictionary] = []
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
if file == null:
_fail("couldn't read %s" % path)
return lines
while not file.eof_reached():
var line: String = file.get_line()
if not line.begins_with("{"):
continue
var parsed: Variant = JSON.parse_string(line)
if parsed is Dictionary:
lines.append(parsed)
return lines
func _first(lines: Array[Dictionary], event: String) -> Dictionary:
for record: Dictionary in lines:
if String(record.get("event", "")) == event:
return record
return {}
func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
var found: Array[Dictionary] = []
for record: Dictionary in lines:
if String(record.get("event", "")) == event:
found.append(record)
return found
func _count(lines: Array[Dictionary], event: String) -> int:
return _all(lines, event).size()