Files
tidewall/tools/soak_report.gd
T
claudeandClaude Opus 5.5 05c7568ec1 N8: 24-client network soak, and the server fixes it found
make soak runs a real server and 24 headless clients in three processes for
two whole matches in real time. Each client reaches the server through its own
UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs
unchanged over a bad link. SoakPlayer drives each client from what a client
knows: it drops, glides, heads for high ground, harvests and tries to build.
tools/soak_report.gd checks world hashes against the server's, errors,
unreadable packets, every client playing every match with its connection
intact, memory between matches, and the p99 tick with the network's share.
make soak-selfcheck proves a drifted client world is caught.

Server changes: --matches=N, --world-hash-every=N, tick timing that includes
after_tick, and memory in match and progress lines.

What the soak found and fixed:
- Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the
  12 ms budget, against a 2.8 ms simulation. Encoded records are now shared
  between clients for the tick, unchanged records skip the field diff, and
  clients are spread over the snapshot interval by handle. p99 is 7.3 ms.
- ENet peers had a timeout limit of 0, which gives up on a peer as soon as any
  reliable message has waited the minimum timeout. Now ENet's default, 32.

Not fixed: one or two clients still lose their connection about 15 s into the
second match's drop in each run (see the issue), so the soak still fails and
N8 stays open.

Refs #39

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 21:56:01 -05:00

228 lines
9.6 KiB
GDScript

## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the
## server ran its matches cleanly and within its tick budget, every client joined and
## played every match with no unreadable packets, every client's copy of the world
## matched the server's wherever both took a hash, and neither the server nor the
## client processes grew in memory from one match to the next.
##
## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH
## --client-logs=PATH[,PATH...] --clients=N --matches=N
##
## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own
## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line.
extends SceneTree
## How much a process's memory may grow from the first match to the last before it
## counts as a leak. Each match makes a new world, so some wobble is normal.
const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024
## Hash comparisons each client must reach per match, so a run that compared nothing
## can't pass.
const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10
var _failures: Array[String] = []
func _initialize() -> void:
var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"}
for arg: String in OS.get_cmdline_user_args():
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
if parts.size() != 2 or not options.has(parts[0]):
printerr("soak_report: unknown argument '%s'" % arg)
quit(2)
return
options[parts[0]] = parts[1]
if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty():
printerr("soak_report: --server-log and --client-logs are needed")
quit(2)
return
var result: Dictionary = check(
_read(String(options["server-log"])),
_read_all(String(options["client-logs"]).split(",")),
int(options["clients"]), int(options["matches"]))
print(JSON.stringify(result))
for failure: String in _failures:
print("soak FAIL: " + failure)
quit(0 if _failures.is_empty() else 1)
## Checks a run from its parsed log lines. Returns the measurements; failures go in
## `_failures`.
func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary:
var result: Dictionary = {"event": "soak"}
# The server.
var stats: Dictionary = _first(server, "stats")
var stop: Dictionary = _first(server, "stop")
if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0:
_fail("the server didn't stop cleanly: %s" % JSON.stringify(stop))
var ended: int = _count(server, "results")
result["matches_ended"] = ended
if ended < matches:
_fail("the server ended %d matches, not %d" % [ended, matches])
var budget: int = int(stats.get("budget_us", 0))
result["tick_p99_us"] = stats.get("with_net_p99_us", -1)
result["tick_max_us"] = stats.get("with_net_max_us", -1)
result["sim_p99_us"] = stats.get("p99_us", -1)
result["over_budget_ticks"] = stats.get("over_budget", -1)
if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget:
_fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget])
result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server")
# The clients.
var reports: Array[Dictionary] = _all(clients, "soak_client")
result["clients"] = reports.size()
if reports.size() != client_count:
_fail("%d clients reported, not %d" % [reports.size(), client_count])
var down: int = 0
var up: int = 0
var played: Dictionary = {}
for record: Dictionary in _all(clients, "client_match"):
played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1
for report: Dictionary in reports:
var number: int = int(report["client"])
down += int(report["down_bytes"])
up += int(report["up_bytes"])
if not report["joined"]:
_fail("client %d never joined" % number)
if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN:
_fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)])
if int(report["bad_packets"]) != 0:
_fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]])
if int(played.get(number, 0)) < matches:
_fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches])
var processes: Array[Dictionary] = _all(clients, "soak_process")
var seconds: int = 0
var mispredictions: int = 0
var reconciles: int = 0
var snaps: int = 0
for record: Dictionary in _all(clients, "client_match"):
mispredictions += int(record["mispredictions"])
reconciles += int(record["reconciles"])
snaps += int(record["snaps"])
var lost: int = 0
var sent: int = 0
for process: Dictionary in processes:
seconds = maxi(seconds, int(process["seconds"]))
lost += int(process["up_lost"]) + int(process["down_lost"])
sent += int(process["up_packets"]) + int(process["down_packets"])
if int(process["errors"]) != 0:
_fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")])
result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0)))
result["client_processes"] = processes.size()
result["seconds"] = seconds
result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001)
if seconds > 0 and not reports.is_empty():
result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1)
result["reconciles"] = reconciles
result["mispredictions"] = mispredictions
result["prediction_snaps"] = snaps
result["client_mem_growth_bytes"] = _client_growth(clients)
# The world, compared hash by hash.
var server_hashes: Dictionary = {}
for record: Dictionary in _all(server, "world_hash"):
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
var compared: int = 0
var differ: int = 0
var per_client: Dictionary = {}
for record: Dictionary in _all(clients, "hash"):
var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"]))
if not server_hashes.has(key):
continue
compared += 1
per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1
if server_hashes[key] != String(record["hash"]):
differ += 1
if differ <= 5:
_fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y])
result["hashes_compared"] = compared
result["hashes_differing"] = differ
if differ > 5:
_fail("%d world hashes differed in all" % differ)
for report: Dictionary in reports:
var number: int = int(report["client"])
if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches:
_fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)])
return result
## How much the server's memory grew from the first `phase` entry it logged to the last.
func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int:
var samples: Array[int] = []
for record: Dictionary in _all(lines, event):
if String(record.get("to", "")) == phase and record.has("mem_bytes"):
samples.append(int(record["mem_bytes"]))
return _judge(samples, who)
## The same for each client process, from its "memory" lines.
func _client_growth(lines: Array[Dictionary]) -> int:
var by_process: Dictionary = {}
for record: Dictionary in lines:
if String(record.get("event", "")) == "memory":
var process: int = int(record.get("_process", 0))
if not by_process.has(process):
by_process[process] = [] as Array[int]
(by_process[process] as Array[int]).append(int(record["mem_bytes"]))
var worst: int = 0
for process: int in by_process:
worst = maxi(worst, _judge(by_process[process], "client process %d" % process))
return worst
func _judge(samples: Array[int], who: String) -> int:
if samples.size() < 2:
_fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()])
return 0
var growth: int = samples[samples.size() - 1] - samples[0]
if growth > LEAK_ALLOWANCE_BYTES:
_fail("%s grew by %d bytes from the first match to the last" % [who, growth])
return growth
func _fail(message: String) -> void:
_failures.append(message)
func _read_all(paths: PackedStringArray) -> Array[Dictionary]:
var lines: Array[Dictionary] = []
for index: int in paths.size():
for record: Dictionary in _read(paths[index]):
record["_process"] = index
lines.append(record)
return lines
## The JSON object lines of a log; anything else in it is skipped.
func _read(path: String) -> Array[Dictionary]:
var lines: Array[Dictionary] = []
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
if file == null:
_fail("couldn't read %s" % path)
return lines
while not file.eof_reached():
var line: String = file.get_line()
if not line.begins_with("{"):
continue
var parsed: Variant = JSON.parse_string(line)
if parsed is Dictionary:
lines.append(parsed)
return lines
func _first(lines: Array[Dictionary], event: String) -> Dictionary:
for record: Dictionary in lines:
if String(record.get("event", "")) == event:
return record
return {}
func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
var found: Array[Dictionary] = []
for record: Dictionary in lines:
if String(record.get("event", "")) == event:
found.append(record)
return found
func _count(lines: Array[Dictionary], event: String) -> int:
return _all(lines, event).size()