When a client's world hash differs from the server's, the soak report now names the part that differs (tide, props, pieces or loot), the client's ClientWorld tick, and each piece whose entry differs on the two sides, so the next transient mismatch can be traced. - NetWorld.world_detail(): part hashes and a piece list. world_hash() is unchanged: the same bytes, hashed in four parts. - The server's world_hash log lines and the soak clients' hash lines carry the detail. A full soak with this (24 clients, 2 matches) compared 3,743 hashes with 0 differing, so the mismatch didn't come back to trace yet. Refs #80 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
275 lines
12 KiB
GDScript
275 lines
12 KiB
GDScript
## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the
|
|
## server ran its matches cleanly and within its tick budget, every client joined and
|
|
## played every match with no unreadable packets, every client's copy of the world
|
|
## matched the server's wherever both took a hash, and neither the server nor the
|
|
## client processes grew in memory from one match to the next.
|
|
##
|
|
## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH
|
|
## --client-logs=PATH[,PATH...] --clients=N --matches=N
|
|
##
|
|
## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own
|
|
## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line.
|
|
extends SceneTree
|
|
|
|
## How much a process's memory may grow from the first match to the last before it
|
|
## counts as a leak. Each match makes a new world, so some wobble is normal.
|
|
const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024
|
|
## Hash comparisons each client must reach per match, so a run that compared nothing
|
|
## can't pass.
|
|
const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10
|
|
|
|
var _failures: Array[String] = []
|
|
|
|
|
|
func _initialize() -> void:
|
|
var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"}
|
|
for arg: String in OS.get_cmdline_user_args():
|
|
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
|
if parts.size() != 2 or not options.has(parts[0]):
|
|
printerr("soak_report: unknown argument '%s'" % arg)
|
|
quit(2)
|
|
return
|
|
options[parts[0]] = parts[1]
|
|
if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty():
|
|
printerr("soak_report: --server-log and --client-logs are needed")
|
|
quit(2)
|
|
return
|
|
var result: Dictionary = check(
|
|
_read(String(options["server-log"])),
|
|
_read_all(String(options["client-logs"]).split(",")),
|
|
int(options["clients"]), int(options["matches"]))
|
|
print(JSON.stringify(result))
|
|
for failure: String in _failures:
|
|
print("soak FAIL: " + failure)
|
|
quit(0 if _failures.is_empty() else 1)
|
|
|
|
|
|
## Checks a run from its parsed log lines. Returns the measurements; failures go in
|
|
## `_failures`.
|
|
func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary:
|
|
var result: Dictionary = {"event": "soak"}
|
|
# The server.
|
|
var stats: Dictionary = _first(server, "stats")
|
|
var stop: Dictionary = _first(server, "stop")
|
|
if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0:
|
|
_fail("the server didn't stop cleanly: %s" % JSON.stringify(stop))
|
|
var ended: int = _count(server, "results")
|
|
result["matches_ended"] = ended
|
|
if ended < matches:
|
|
_fail("the server ended %d matches, not %d" % [ended, matches])
|
|
var budget: int = int(stats.get("budget_us", 0))
|
|
result["tick_p99_us"] = stats.get("with_net_p99_us", -1)
|
|
result["tick_max_us"] = stats.get("with_net_max_us", -1)
|
|
result["sim_p99_us"] = stats.get("p99_us", -1)
|
|
result["over_budget_ticks"] = stats.get("over_budget", -1)
|
|
if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget:
|
|
_fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget])
|
|
result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server")
|
|
# The clients.
|
|
var reports: Array[Dictionary] = _all(clients, "soak_client")
|
|
result["clients"] = reports.size()
|
|
if reports.size() != client_count:
|
|
_fail("%d clients reported, not %d" % [reports.size(), client_count])
|
|
var down: int = 0
|
|
var up: int = 0
|
|
var played: Dictionary = {}
|
|
for record: Dictionary in _all(clients, "client_match"):
|
|
played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1
|
|
for report: Dictionary in reports:
|
|
var number: int = int(report["client"])
|
|
down += int(report["down_bytes"])
|
|
up += int(report["up_bytes"])
|
|
if not report["joined"]:
|
|
_fail("client %d never joined" % number)
|
|
if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN:
|
|
_fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)])
|
|
if int(report["bad_packets"]) != 0:
|
|
_fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]])
|
|
if int(played.get(number, 0)) < matches:
|
|
_fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches])
|
|
var processes: Array[Dictionary] = _all(clients, "soak_process")
|
|
var seconds: int = 0
|
|
var mispredictions: int = 0
|
|
var reconciles: int = 0
|
|
var snaps: int = 0
|
|
for record: Dictionary in _all(clients, "client_match"):
|
|
mispredictions += int(record["mispredictions"])
|
|
reconciles += int(record["reconciles"])
|
|
snaps += int(record["snaps"])
|
|
var lost: int = 0
|
|
var sent: int = 0
|
|
for process: Dictionary in processes:
|
|
seconds = maxi(seconds, int(process["seconds"]))
|
|
lost += int(process["up_lost"]) + int(process["down_lost"])
|
|
sent += int(process["up_packets"]) + int(process["down_packets"])
|
|
if int(process["errors"]) != 0:
|
|
_fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")])
|
|
result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0)))
|
|
result["client_processes"] = processes.size()
|
|
result["seconds"] = seconds
|
|
result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001)
|
|
if seconds > 0 and not reports.is_empty():
|
|
result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
|
result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
|
result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1)
|
|
result["reconciles"] = reconciles
|
|
result["mispredictions"] = mispredictions
|
|
result["prediction_snaps"] = snaps
|
|
result["client_mem_growth_bytes"] = _client_growth(clients)
|
|
# The world, compared hash by hash.
|
|
var server_hashes: Dictionary = {}
|
|
var server_detail: Dictionary = {}
|
|
for record: Dictionary in _all(server, "world_hash"):
|
|
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
|
|
server_detail[Vector2i(int(record["match"]), int(record["sim_tick"]))] = record.get("detail", {})
|
|
var compared: int = 0
|
|
var differ: int = 0
|
|
var per_client: Dictionary = {}
|
|
for record: Dictionary in _all(clients, "hash"):
|
|
var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"]))
|
|
if not server_hashes.has(key):
|
|
continue
|
|
compared += 1
|
|
per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1
|
|
if server_hashes[key] != String(record["hash"]):
|
|
differ += 1
|
|
if differ <= 5:
|
|
_fail("client %d's world differs from the server's in match %d at tick %d (replica tick %s): %s" % [record["client"], key.x, key.y, record.get("replica_tick", "?"), describe_difference(server_detail.get(key, {}), record.get("detail", {}))])
|
|
result["hashes_compared"] = compared
|
|
result["hashes_differing"] = differ
|
|
if differ > 5:
|
|
_fail("%d world hashes differed in all" % differ)
|
|
for report: Dictionary in reports:
|
|
var number: int = int(report["client"])
|
|
if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches:
|
|
_fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)])
|
|
return result
|
|
|
|
|
|
## How much the server's memory grew from the first `phase` entry it logged to the last.
|
|
func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int:
|
|
var samples: Array[int] = []
|
|
for record: Dictionary in _all(lines, event):
|
|
if String(record.get("to", "")) == phase and record.has("mem_bytes"):
|
|
samples.append(int(record["mem_bytes"]))
|
|
return _judge(samples, who)
|
|
|
|
|
|
## The same for each client process, from its "memory" lines.
|
|
func _client_growth(lines: Array[Dictionary]) -> int:
|
|
var by_process: Dictionary = {}
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == "memory":
|
|
var process: int = int(record.get("_process", 0))
|
|
if not by_process.has(process):
|
|
by_process[process] = [] as Array[int]
|
|
(by_process[process] as Array[int]).append(int(record["mem_bytes"]))
|
|
var worst: int = 0
|
|
for process: int in by_process:
|
|
worst = maxi(worst, _judge(by_process[process], "client process %d" % process))
|
|
# A minute into each match, every island alive must be a client's world or one being
|
|
# made. Anything more is an island nothing should still hold (#39).
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) != "memory_late":
|
|
continue
|
|
var extra: int = int((record.get("worlds", {}) as Dictionary).get("extra_islands", 0))
|
|
if extra > 0:
|
|
_fail("client process %d still held %d islands beyond its clients' worlds in match %d" % [int(record.get("_process", 0)), extra, int(record.get("match", -1))])
|
|
return worst
|
|
|
|
|
|
func _judge(samples: Array[int], who: String) -> int:
|
|
if samples.size() < 2:
|
|
_fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()])
|
|
return 0
|
|
var growth: int = samples[samples.size() - 1] - samples[0]
|
|
if growth > LEAK_ALLOWANCE_BYTES:
|
|
_fail("%s grew by %d bytes from the first match to the last" % [who, growth])
|
|
return growth
|
|
|
|
|
|
func _fail(message: String) -> void:
|
|
_failures.append(message)
|
|
|
|
|
|
func _read_all(paths: PackedStringArray) -> Array[Dictionary]:
|
|
var lines: Array[Dictionary] = []
|
|
for index: int in paths.size():
|
|
for record: Dictionary in _read(paths[index]):
|
|
record["_process"] = index
|
|
lines.append(record)
|
|
return lines
|
|
|
|
|
|
## The JSON object lines of a log; anything else in it is skipped.
|
|
func _read(path: String) -> Array[Dictionary]:
|
|
var lines: Array[Dictionary] = []
|
|
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
|
|
if file == null:
|
|
_fail("couldn't read %s" % path)
|
|
return lines
|
|
while not file.eof_reached():
|
|
var line: String = file.get_line()
|
|
if not line.begins_with("{"):
|
|
continue
|
|
var parsed: Variant = JSON.parse_string(line)
|
|
if parsed is Dictionary:
|
|
lines.append(parsed)
|
|
return lines
|
|
|
|
|
|
func _first(lines: Array[Dictionary], event: String) -> Dictionary:
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == event:
|
|
return record
|
|
return {}
|
|
|
|
|
|
func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
|
|
var found: Array[Dictionary] = []
|
|
for record: Dictionary in lines:
|
|
if String(record.get("event", "")) == event:
|
|
found.append(record)
|
|
return found
|
|
|
|
|
|
func _count(lines: Array[Dictionary], event: String) -> int:
|
|
return _all(lines, event).size()
|
|
|
|
|
|
## Which parts of two NetWorld.world_detail() records differ, and for the pieces, each
|
|
## piece whose entry differs, as "id: server [...] client [...]" (at most eight).
|
|
static func describe_difference(server: Dictionary, client: Dictionary) -> String:
|
|
if server.is_empty() or client.is_empty():
|
|
return "no detail logged"
|
|
var parts: Array[String] = []
|
|
for name: String in ["tide", "props", "pieces", "loot"]:
|
|
if String(server.get(name, "")) != String(client.get(name, "")):
|
|
parts.append(name)
|
|
var notes: Array[String] = ["parts differing: %s" % (", ".join(parts) if not parts.is_empty() else "none")]
|
|
var theirs: Dictionary = {}
|
|
for entry: Array in client.get("piece_list", []):
|
|
theirs[int(entry[0])] = entry
|
|
var ours: Dictionary = {}
|
|
for entry: Array in server.get("piece_list", []):
|
|
ours[int(entry[0])] = entry
|
|
var ids: Dictionary = {}
|
|
for piece_id: int in ours:
|
|
ids[piece_id] = true
|
|
for piece_id: int in theirs:
|
|
ids[piece_id] = true
|
|
var sorted: Array = ids.keys()
|
|
sorted.sort()
|
|
var shown: int = 0
|
|
for piece_id: int in sorted:
|
|
var server_entry: Variant = ours.get(piece_id)
|
|
var client_entry: Variant = theirs.get(piece_id)
|
|
if server_entry != null and client_entry != null and str(server_entry) == str(client_entry):
|
|
continue
|
|
shown += 1
|
|
if shown <= 8:
|
|
notes.append("piece %d: server %s client %s" % [piece_id, str(server_entry), str(client_entry)])
|
|
if shown > 8:
|
|
notes.append("and %d more pieces" % (shown - 8))
|
|
return "; ".join(notes)
|