When a client's world hash differs from the server's, the soak report now names the part that differs (tide, props, pieces or loot), the client's ClientWorld tick, and each piece whose entry differs on the two sides, so the next transient mismatch can be traced. - NetWorld.world_detail(): part hashes and a piece list. world_hash() is unchanged: the same bytes, hashed in four parts. - The server's world_hash log lines and the soak clients' hash lines carry the detail. A full soak with this (24 clients, 2 matches) compared 3,743 hashes with 0 differing, so the mismatch didn't come back to trace yet. Refs #80 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
258 lines
12 KiB
GDScript
258 lines
12 KiB
GDScript
## Runs several headless network clients in one process for the soak test (make soak).
|
|
## Each client reaches the server through its own UdpRelay, which delays, jitters and
|
|
## loses datagrams both ways, and is played by a SoakPlayer.
|
|
##
|
|
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
|
|
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
|
|
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
|
|
##
|
|
## --tamper=N knocks a point off one prop's health in client N's world just before its
|
|
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
|
|
## client whose world has drifted is caught.
|
|
##
|
|
## Prints JSON lines:
|
|
## {"event":"hash", "client", "match", "sim_tick", "hash", "detail", "replica_tick"}: the
|
|
## client's world hash (with NetWorld.world_detail() and its ClientWorld's tick) on
|
|
## the snapshots where the server logs its own (--world-hash-every on the server, the
|
|
## same N as --hash-every here), when the client's world has caught up exactly.
|
|
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
|
|
## for a match it played, once the match is over for it.
|
|
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
|
|
## {"event":"soak_client", ...}: one per client at the end.
|
|
## {"event":"soak_process", ...}: frame times, totals and errors, last.
|
|
## It stops when every client's connection has closed, or after --seconds.
|
|
##
|
|
## Making a match's island takes most of a second. Each client makes it on a worker
|
|
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
|
|
## keeps servicing every connection while islands are made.
|
|
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
|
|
extends SceneTree
|
|
|
|
const FPS: int = 60
|
|
const LATE_MEMORY_MSEC: int = 60000
|
|
|
|
var _clients: Array[Dictionary] = []
|
|
var _errors: ErrorCounter = ErrorCounter.new()
|
|
var _hash_every: int = 300
|
|
var _deadline_msec: int = 0
|
|
var _frame_usec: PackedInt64Array = PackedInt64Array()
|
|
var _memory_logged: Dictionary = {}
|
|
var _started_msec: int = 0
|
|
var _done: bool = false
|
|
var _tamper: int = -1
|
|
|
|
|
|
func _initialize() -> void:
|
|
OS.add_logger(_errors)
|
|
var options: Dictionary = {
|
|
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
|
|
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
|
|
}
|
|
for arg: String in OS.get_cmdline_user_args():
|
|
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
|
if parts.size() != 2 or not options.has(parts[0]):
|
|
printerr("soak_clients: unknown argument '%s'" % arg)
|
|
quit(2)
|
|
return
|
|
options[parts[0]] = parts[1]
|
|
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
|
|
if target.is_empty():
|
|
printerr("soak_clients: --server=HOST:PORT is needed")
|
|
quit(2)
|
|
return
|
|
Engine.max_fps = FPS
|
|
_hash_every = int(options["hash-every"])
|
|
_tamper = int(options["tamper"])
|
|
_started_msec = Time.get_ticks_msec()
|
|
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
|
|
var first: int = int(options["first"])
|
|
for index: int in int(options["clients"]):
|
|
var number: int = first + index
|
|
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
|
|
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
|
|
if error != OK:
|
|
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
|
|
quit(2)
|
|
return
|
|
var client: Dictionary = {
|
|
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
|
|
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
|
|
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
|
|
}
|
|
var game: ClientGame = client["game"]
|
|
var player: SoakPlayer = client["player"]
|
|
game.sample_input = func(tick: int) -> PlayerCommand:
|
|
if game.world == null:
|
|
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
|
|
return player.command(tick, game.prediction.player, game.world)
|
|
game.on_snapshot = _sample.bind(client)
|
|
game.build_in_background = true
|
|
game.attach(client["link"])
|
|
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
|
|
_clients.append(client)
|
|
|
|
|
|
func _process(delta: float) -> bool:
|
|
if _done:
|
|
return true
|
|
var began: int = Time.get_ticks_usec()
|
|
var open: int = 0
|
|
for client: Dictionary in _clients:
|
|
var relay: UdpRelay = client["relay"]
|
|
var link: NetClient = client["link"]
|
|
var game: ClientGame = client["game"]
|
|
relay.poll()
|
|
link.poll()
|
|
for packet: Dictionary in link.packets:
|
|
game.receive(int(packet["channel"]), packet["bytes"])
|
|
game.advance(delta)
|
|
relay.poll()
|
|
_follow(client)
|
|
if link.state != NetClient.CLOSED:
|
|
open += 1
|
|
_frame_usec.append(Time.get_ticks_usec() - began)
|
|
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
|
|
_finish()
|
|
return true
|
|
return false
|
|
|
|
|
|
## How many distinct worlds this process's clients still reach (their own, the ground
|
|
## their prediction runs on, and islands made in the background but not yet taken
|
|
## up), and how many Simulations and grounds are alive in all. `extra_islands` counts
|
|
## live Simulations nothing here accounts for: a leak (#39).
|
|
func _live_worlds() -> Dictionary:
|
|
var worlds: Dictionary = {}
|
|
var grounds: Dictionary = {}
|
|
var built: int = 0
|
|
for client: Dictionary in _clients:
|
|
var game: ClientGame = client["game"]
|
|
if game.world != null:
|
|
worlds[game.world.get_instance_id()] = true
|
|
var ground: SimGround = game.prediction.get("_ground")
|
|
if ground != null:
|
|
grounds[ground.get_instance_id()] = true
|
|
var made: Simulation = game.get("_built")
|
|
if made != null and made != game.world:
|
|
built += 1
|
|
if game.building_world():
|
|
built += 1
|
|
var live: int = LiveCount.of(&"Simulation")
|
|
return {"worlds": worlds.size(), "prediction_grounds": grounds.size(), "built": built,
|
|
"live_simulations": live, "live_grounds": LiveCount.of(&"SimGround"), "extra_islands": maxi(0, live - worlds.size() - built)}
|
|
|
|
|
|
## Keeps track of a client's phases, and reports each match it played once it's over.
|
|
func _follow(client: Dictionary) -> void:
|
|
var game: ClientGame = client["game"]
|
|
var link: NetClient = client["link"]
|
|
if link.state == NetClient.CLOSED and not client.has("closed_s"):
|
|
# Say at once when a connection closes, with what its relay last saw each way.
|
|
var relay: UdpRelay = client["relay"]
|
|
var now: int = Time.get_ticks_msec()
|
|
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
|
|
print(JSON.stringify({
|
|
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
|
|
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
|
|
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
|
|
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
|
|
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
|
|
}))
|
|
if game.world == null:
|
|
return
|
|
var phases: Array = client["phases"]
|
|
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
|
|
phases.append(game.match_phase)
|
|
if game.playing():
|
|
client["frames_playing"] = int(client["frames_playing"]) + 1
|
|
client["stats_match"] = game.match_index
|
|
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
|
|
elif int(client["stats_match"]) >= 0:
|
|
_report_match(client)
|
|
# Once a match is in play every client here has let go of the last match's world.
|
|
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
|
|
_memory_logged[game.match_index] = Time.get_ticks_msec()
|
|
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
|
|
elif game.match_phase == Match.PLAY and _memory_logged.has(game.match_index) and _memory_logged.has(-1 - game.match_index) == false \
|
|
and Time.get_ticks_msec() - int(_memory_logged[game.match_index]) > LATE_MEMORY_MSEC:
|
|
# A second look a minute later, for telling a leak from something let go late.
|
|
_memory_logged[-1 - game.match_index] = true
|
|
print(JSON.stringify({"event": "memory_late", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
|
|
|
|
|
|
func _report_match(client: Dictionary) -> void:
|
|
var stats: Array = client["last_stats"]
|
|
var record: Dictionary = {
|
|
"event": "client_match", "client": client["number"], "match": client["stats_match"],
|
|
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
|
|
}
|
|
(client["matches"] as Array).append(record)
|
|
print(JSON.stringify(record))
|
|
client["stats_match"] = -1
|
|
|
|
|
|
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
|
|
## world has caught up to exactly what the snapshot says the server had sent.
|
|
func _sample(client: Dictionary) -> void:
|
|
var game: ClientGame = client["game"]
|
|
var tick: int = game.receiver.sim_tick
|
|
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
|
|
# so it names more than one state; the server doesn't log it either.
|
|
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
|
return
|
|
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
|
|
client["skipped"] = int(client["skipped"]) + 1
|
|
return
|
|
client["samples"] = int(client["samples"]) + 1
|
|
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
|
|
_tamper = -1
|
|
game.world.props.health[0] -= 1
|
|
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world), "detail": NetWorld.world_detail(game.world), "replica_tick": game.replica.tick}))
|
|
|
|
|
|
func _finish() -> void:
|
|
_done = true
|
|
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
|
|
for client: Dictionary in _clients:
|
|
var relay: UdpRelay = client["relay"]
|
|
var link: NetClient = client["link"]
|
|
var game: ClientGame = client["game"]
|
|
if int(client["stats_match"]) >= 0:
|
|
_report_match(client)
|
|
var phase_names: Array[String] = []
|
|
for phase: int in client["phases"]:
|
|
phase_names.append(Match.PHASE_NAMES[phase])
|
|
print(JSON.stringify({
|
|
"event": "soak_client", "client": client["number"], "name": link.player_name,
|
|
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
|
|
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
|
|
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
|
|
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
|
|
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
|
|
"world_gaps": game.replica.gaps,
|
|
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
|
|
}))
|
|
for key: String in totals:
|
|
totals[key] = int(totals[key]) + int(relay.get(key))
|
|
link.leave()
|
|
relay.close()
|
|
# The input and snapshot callables hold the client: let go of them, or nothing frees.
|
|
game.sample_input = Callable()
|
|
game.on_snapshot = Callable()
|
|
var sorted: PackedInt64Array = _frame_usec.duplicate()
|
|
sorted.sort()
|
|
var summary: Dictionary = {
|
|
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
|
|
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
|
|
}
|
|
if not sorted.is_empty():
|
|
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
|
|
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
|
|
summary.merge(totals)
|
|
if _errors.count() > 0:
|
|
summary["first_error"] = _errors.first()
|
|
print(JSON.stringify(summary))
|
|
_clients.clear()
|
|
OS.remove_logger(_errors)
|
|
quit(1 if _errors.count() > 0 else 0)
|