Files
tidewall/tools/soak_clients.gd
T
claudeandClaude Opus 5.5 05c7568ec1 N8: 24-client network soak, and the server fixes it found
make soak runs a real server and 24 headless clients in three processes for
two whole matches in real time. Each client reaches the server through its own
UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs
unchanged over a bad link. SoakPlayer drives each client from what a client
knows: it drops, glides, heads for high ground, harvests and tries to build.
tools/soak_report.gd checks world hashes against the server's, errors,
unreadable packets, every client playing every match with its connection
intact, memory between matches, and the p99 tick with the network's share.
make soak-selfcheck proves a drifted client world is caught.

Server changes: --matches=N, --world-hash-every=N, tick timing that includes
after_tick, and memory in match and progress lines.

What the soak found and fixed:
- Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the
  12 ms budget, against a 2.8 ms simulation. Encoded records are now shared
  between clients for the tick, unchanged records skip the field diff, and
  clients are spread over the snapshot interval by handle. p99 is 7.3 ms.
- ENet peers had a timeout limit of 0, which gives up on a peer as soon as any
  reliable message has waited the minimum timeout. Now ENet's default, 32.

Not fixed: one or two clients still lose their connection about 15 s into the
second match's drop in each run (see the issue), so the soak still fails and
N8 stays open.

Refs #39

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 21:56:01 -05:00

277 lines
12 KiB
GDScript

## Runs several headless network clients in one process for the soak test (make soak).
## Each client reaches the server through its own UdpRelay, which delays, jitters and
## loses datagrams both ways, and is played by a SoakPlayer.
##
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
##
## --tamper=N knocks a point off one prop's health in client N's world just before its
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
## client whose world has drifted is caught.
##
## Prints JSON lines:
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
## the snapshots where the server logs its own (--world-hash-every on the server, the
## same N as --hash-every here), when the client's world has caught up exactly.
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
## for a match it played, once the match is over for it.
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
## {"event":"soak_client", ...}: one per client at the end.
## {"event":"soak_process", ...}: frame times, totals and errors, last.
## It stops when every client's connection has closed, or after --seconds.
##
## Making a match's island takes most of a second. A real player does it once, but a
## dozen clients here would all do it in the same frame and starve every connection of
## service for many seconds. So a MATCH notice that brings a new match is held back,
## with everything after it for that client, until the driver has made that client's
## world, and the driver makes at most one world a frame.
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
extends SceneTree
const FPS: int = 60
var _clients: Array[Dictionary] = []
var _errors: ErrorCounter = ErrorCounter.new()
var _hash_every: int = 300
var _deadline_msec: int = 0
var _frame_usec: PackedInt64Array = PackedInt64Array()
var _memory_logged: Dictionary = {}
var _started_msec: int = 0
var _done: bool = false
var _tamper: int = -1
func _initialize() -> void:
OS.add_logger(_errors)
var options: Dictionary = {
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
}
for arg: String in OS.get_cmdline_user_args():
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
if parts.size() != 2 or not options.has(parts[0]):
printerr("soak_clients: unknown argument '%s'" % arg)
quit(2)
return
options[parts[0]] = parts[1]
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
if target.is_empty():
printerr("soak_clients: --server=HOST:PORT is needed")
quit(2)
return
Engine.max_fps = FPS
_hash_every = int(options["hash-every"])
_tamper = int(options["tamper"])
_started_msec = Time.get_ticks_msec()
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
var first: int = int(options["first"])
for index: int in int(options["clients"]):
var number: int = first + index
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
if error != OK:
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
quit(2)
return
var client: Dictionary = {
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
"held": [], "notice": "", "ready_world": null, "ready_seed": 0,
}
var game: ClientGame = client["game"]
var player: SoakPlayer = client["player"]
game.sample_input = func(tick: int) -> PlayerCommand:
if game.world == null:
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
return player.command(tick, game.prediction.player, game.world)
game.on_snapshot = _sample.bind(client)
game.make_world = func(match_seed: int) -> Simulation:
var ready: Simulation = client["ready_world"]
client["ready_world"] = null
if ready != null and int(client["ready_seed"]) == match_seed:
return ready
return Simulation.new(match_seed)
game.attach(client["link"])
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
_clients.append(client)
func _process(delta: float) -> bool:
if _done:
return true
var began: int = Time.get_ticks_usec()
var open: int = 0
var may_make_world: bool = true
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
relay.poll()
link.poll()
_take_packets(client, link.packets)
if may_make_world and not (client["held"] as Array).is_empty():
may_make_world = false
_make_world(client)
game.advance(delta)
relay.poll()
_follow(client)
if link.state != NetClient.CLOSED:
open += 1
_frame_usec.append(Time.get_ticks_usec() - began)
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
_finish()
return true
return false
## Passes packets to the client's game, holding back a MATCH notice for a new match,
## and everything after it, until its world has been made.
func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void:
var game: ClientGame = client["game"]
var held: Array = client["held"]
for packet: Dictionary in packets:
if held.is_empty() and not _brings_new_match(client, packet):
game.receive(int(packet["channel"]), packet["bytes"])
else:
held.append(packet)
## True for a MATCH notice of a match (index and seed) the client hasn't been told of.
func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool:
var bytes: PackedByteArray = packet["bytes"]
if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
return false
var notice: Dictionary = NetSnapshot.decode_match(bytes)
if notice.is_empty():
return false
var key: String = "%d:%d" % [notice["index"], notice["seed"]]
if key == String(client["notice"]):
return false
client["notice"] = key
client["ready_seed"] = int(notice["seed"])
return true
## Makes the world for the held notice, then passes on everything held.
func _make_world(client: Dictionary) -> void:
var game: ClientGame = client["game"]
client["ready_world"] = Simulation.new(int(client["ready_seed"]))
var held: Array = client["held"]
client["held"] = []
for packet: Dictionary in held:
game.receive(int(packet["channel"]), packet["bytes"])
## Keeps track of a client's phases, and reports each match it played once it's over.
func _follow(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var link: NetClient = client["link"]
if link.state == NetClient.CLOSED and not client.has("closed_s"):
# Say at once when a connection closes, with what its relay last saw each way.
var relay: UdpRelay = client["relay"]
var now: int = Time.get_ticks_msec()
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
print(JSON.stringify({
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
"held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(),
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
}))
if game.world == null:
return
var phases: Array = client["phases"]
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
phases.append(game.match_phase)
if game.playing():
client["frames_playing"] = int(client["frames_playing"]) + 1
client["stats_match"] = game.match_index
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
elif int(client["stats_match"]) >= 0:
_report_match(client)
# Once a match is in play every client here has let go of the last match's world.
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
_memory_logged[game.match_index] = true
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage()}))
func _report_match(client: Dictionary) -> void:
var stats: Array = client["last_stats"]
var record: Dictionary = {
"event": "client_match", "client": client["number"], "match": client["stats_match"],
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
}
(client["matches"] as Array).append(record)
print(JSON.stringify(record))
client["stats_match"] = -1
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
## world has caught up to exactly what the snapshot says the server had sent.
func _sample(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var tick: int = game.receiver.sim_tick
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
# so it names more than one state; the server doesn't log it either.
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
return
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
client["skipped"] = int(client["skipped"]) + 1
return
client["samples"] = int(client["samples"]) + 1
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
_tamper = -1
game.world.props.health[0] -= 1
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
func _finish() -> void:
_done = true
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
if int(client["stats_match"]) >= 0:
_report_match(client)
var phase_names: Array[String] = []
for phase: int in client["phases"]:
phase_names.append(Match.PHASE_NAMES[phase])
print(JSON.stringify({
"event": "soak_client", "client": client["number"], "name": link.player_name,
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
"world_gaps": game.replica.gaps,
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
}))
for key: String in totals:
totals[key] = int(totals[key]) + int(relay.get(key))
link.leave()
relay.close()
# The input and snapshot callables hold the client: let go of them, or nothing frees.
game.sample_input = Callable()
game.on_snapshot = Callable()
game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed)
client["ready_world"] = null
var sorted: PackedInt64Array = _frame_usec.duplicate()
sorted.sort()
var summary: Dictionary = {
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
}
if not sorted.is_empty():
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
summary.merge(totals)
if _errors.count() > 0:
summary["first_error"] = _errors.first()
print(JSON.stringify(summary))
_clients.clear()
OS.remove_logger(_errors)
quit(1 if _errors.count() > 0 else 0)