Files
tidewall/tools/soak_clients.gd
T
claudeandClaude Opus 5.5 09d663e647 N8: Keep connections serviced while islands are made, and stop servers gracefully
The soak's drops under loss came from stalls: the server's island (#79) and
each client process making islands on its main thread fed ENet second-long
round-trip samples, its retransmission timer grew, and two losses of one
reliable message in the busy drop that followed reached the 15 s timeout.

- ClientGame.build_in_background makes a new match's island on a
  WorkerThreadPool task and holds the packets that arrive meanwhile, then
  takes them in order. The networked client and the soak's clients use it;
  the soak's own one-island-a-frame holding is gone.
- NetServer.close(grace_msec) sends the shutdown notice reliably and
  services the socket while clients acknowledge it. The headless server
  waits up to 2 s. A notice sent once was lost under 2% loss, and that
  client timed out instead of hearing the server stop.
- Tests: background island holding and ordering, freeing mid-build, a
  graceful shutdown reaching four clients behind 50% loss (it fails with
  the old one-shot notice), and the scripted UDP match and client capture
  wait for the client's island the way a real 20 s lobby would.

The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches,
with 0 of 3,758 world hashes differing and every client told of the stop.
It still fails on client memory growth between matches in two of three
processes (17 and 20 MB), which is not yet explained, so #39 stays open.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 01:28:08 -05:00

226 lines
10 KiB
GDScript

## Runs several headless network clients in one process for the soak test (make soak).
## Each client reaches the server through its own UdpRelay, which delays, jitters and
## loses datagrams both ways, and is played by a SoakPlayer.
##
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
##
## --tamper=N knocks a point off one prop's health in client N's world just before its
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
## client whose world has drifted is caught.
##
## Prints JSON lines:
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
## the snapshots where the server logs its own (--world-hash-every on the server, the
## same N as --hash-every here), when the client's world has caught up exactly.
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
## for a match it played, once the match is over for it.
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
## {"event":"soak_client", ...}: one per client at the end.
## {"event":"soak_process", ...}: frame times, totals and errors, last.
## It stops when every client's connection has closed, or after --seconds.
##
## Making a match's island takes most of a second. Each client makes it on a worker
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
## keeps servicing every connection while islands are made.
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
extends SceneTree
const FPS: int = 60
var _clients: Array[Dictionary] = []
var _errors: ErrorCounter = ErrorCounter.new()
var _hash_every: int = 300
var _deadline_msec: int = 0
var _frame_usec: PackedInt64Array = PackedInt64Array()
var _memory_logged: Dictionary = {}
var _started_msec: int = 0
var _done: bool = false
var _tamper: int = -1
func _initialize() -> void:
OS.add_logger(_errors)
var options: Dictionary = {
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
}
for arg: String in OS.get_cmdline_user_args():
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
if parts.size() != 2 or not options.has(parts[0]):
printerr("soak_clients: unknown argument '%s'" % arg)
quit(2)
return
options[parts[0]] = parts[1]
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
if target.is_empty():
printerr("soak_clients: --server=HOST:PORT is needed")
quit(2)
return
Engine.max_fps = FPS
_hash_every = int(options["hash-every"])
_tamper = int(options["tamper"])
_started_msec = Time.get_ticks_msec()
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
var first: int = int(options["first"])
for index: int in int(options["clients"]):
var number: int = first + index
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
if error != OK:
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
quit(2)
return
var client: Dictionary = {
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
}
var game: ClientGame = client["game"]
var player: SoakPlayer = client["player"]
game.sample_input = func(tick: int) -> PlayerCommand:
if game.world == null:
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
return player.command(tick, game.prediction.player, game.world)
game.on_snapshot = _sample.bind(client)
game.build_in_background = true
game.attach(client["link"])
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
_clients.append(client)
func _process(delta: float) -> bool:
if _done:
return true
var began: int = Time.get_ticks_usec()
var open: int = 0
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
relay.poll()
link.poll()
for packet: Dictionary in link.packets:
game.receive(int(packet["channel"]), packet["bytes"])
game.advance(delta)
relay.poll()
_follow(client)
if link.state != NetClient.CLOSED:
open += 1
_frame_usec.append(Time.get_ticks_usec() - began)
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
_finish()
return true
return false
## Keeps track of a client's phases, and reports each match it played once it's over.
func _follow(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var link: NetClient = client["link"]
if link.state == NetClient.CLOSED and not client.has("closed_s"):
# Say at once when a connection closes, with what its relay last saw each way.
var relay: UdpRelay = client["relay"]
var now: int = Time.get_ticks_msec()
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
print(JSON.stringify({
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
}))
if game.world == null:
return
var phases: Array = client["phases"]
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
phases.append(game.match_phase)
if game.playing():
client["frames_playing"] = int(client["frames_playing"]) + 1
client["stats_match"] = game.match_index
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
elif int(client["stats_match"]) >= 0:
_report_match(client)
# Once a match is in play every client here has let go of the last match's world.
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
_memory_logged[game.match_index] = true
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage()}))
func _report_match(client: Dictionary) -> void:
var stats: Array = client["last_stats"]
var record: Dictionary = {
"event": "client_match", "client": client["number"], "match": client["stats_match"],
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
}
(client["matches"] as Array).append(record)
print(JSON.stringify(record))
client["stats_match"] = -1
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
## world has caught up to exactly what the snapshot says the server had sent.
func _sample(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var tick: int = game.receiver.sim_tick
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
# so it names more than one state; the server doesn't log it either.
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
return
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
client["skipped"] = int(client["skipped"]) + 1
return
client["samples"] = int(client["samples"]) + 1
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
_tamper = -1
game.world.props.health[0] -= 1
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
func _finish() -> void:
_done = true
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
if int(client["stats_match"]) >= 0:
_report_match(client)
var phase_names: Array[String] = []
for phase: int in client["phases"]:
phase_names.append(Match.PHASE_NAMES[phase])
print(JSON.stringify({
"event": "soak_client", "client": client["number"], "name": link.player_name,
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
"world_gaps": game.replica.gaps,
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
}))
for key: String in totals:
totals[key] = int(totals[key]) + int(relay.get(key))
link.leave()
relay.close()
# The input and snapshot callables hold the client: let go of them, or nothing frees.
game.sample_input = Callable()
game.on_snapshot = Callable()
var sorted: PackedInt64Array = _frame_usec.duplicate()
sorted.sort()
var summary: Dictionary = {
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
}
if not sorted.is_empty():
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
summary.merge(totals)
if _errors.count() > 0:
summary["first_error"] = _errors.first()
print(JSON.stringify(summary))
_clients.clear()
OS.remove_logger(_errors)
quit(1 if _errors.count() > 0 else 0)