Files
tidewall/tools/soak_clients.gd
T
claudeandClaude Opus 5.5 880e001fbb N8: Stop the soak player holding old islands, and count live islands
The soak's client memory growth (17 to 20 MB, about five islands, in some
processes) came from SoakPlayer keeping a strong reference to the last world it
played on. It let go only when its client next asked it for a command on the
new world, so finished islands stayed alive across a match change. It now keeps
only the island's seed.

LiveCount counts live Simulations and SimGrounds, which are made on worker
threads too, so it is thread-safe. The soak clients log the counts, and the
soak report fails if, a minute into a match, any island is alive that isn't a
client's world or one being made. That makes a leak name itself instead of
showing up as bytes.

Refs #39

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-13 09:01:49 -05:00

257 lines
12 KiB
GDScript

## Runs several headless network clients in one process for the soak test (make soak).
## Each client reaches the server through its own UdpRelay, which delays, jitters and
## loses datagrams both ways, and is played by a SoakPlayer.
##
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
##
## --tamper=N knocks a point off one prop's health in client N's world just before its
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
## client whose world has drifted is caught.
##
## Prints JSON lines:
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
## the snapshots where the server logs its own (--world-hash-every on the server, the
## same N as --hash-every here), when the client's world has caught up exactly.
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
## for a match it played, once the match is over for it.
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
## {"event":"soak_client", ...}: one per client at the end.
## {"event":"soak_process", ...}: frame times, totals and errors, last.
## It stops when every client's connection has closed, or after --seconds.
##
## Making a match's island takes most of a second. Each client makes it on a worker
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
## keeps servicing every connection while islands are made.
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
extends SceneTree
const FPS: int = 60
const LATE_MEMORY_MSEC: int = 60000
var _clients: Array[Dictionary] = []
var _errors: ErrorCounter = ErrorCounter.new()
var _hash_every: int = 300
var _deadline_msec: int = 0
var _frame_usec: PackedInt64Array = PackedInt64Array()
var _memory_logged: Dictionary = {}
var _started_msec: int = 0
var _done: bool = false
var _tamper: int = -1
func _initialize() -> void:
OS.add_logger(_errors)
var options: Dictionary = {
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
}
for arg: String in OS.get_cmdline_user_args():
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
if parts.size() != 2 or not options.has(parts[0]):
printerr("soak_clients: unknown argument '%s'" % arg)
quit(2)
return
options[parts[0]] = parts[1]
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
if target.is_empty():
printerr("soak_clients: --server=HOST:PORT is needed")
quit(2)
return
Engine.max_fps = FPS
_hash_every = int(options["hash-every"])
_tamper = int(options["tamper"])
_started_msec = Time.get_ticks_msec()
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
var first: int = int(options["first"])
for index: int in int(options["clients"]):
var number: int = first + index
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
if error != OK:
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
quit(2)
return
var client: Dictionary = {
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
}
var game: ClientGame = client["game"]
var player: SoakPlayer = client["player"]
game.sample_input = func(tick: int) -> PlayerCommand:
if game.world == null:
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
return player.command(tick, game.prediction.player, game.world)
game.on_snapshot = _sample.bind(client)
game.build_in_background = true
game.attach(client["link"])
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
_clients.append(client)
func _process(delta: float) -> bool:
if _done:
return true
var began: int = Time.get_ticks_usec()
var open: int = 0
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
relay.poll()
link.poll()
for packet: Dictionary in link.packets:
game.receive(int(packet["channel"]), packet["bytes"])
game.advance(delta)
relay.poll()
_follow(client)
if link.state != NetClient.CLOSED:
open += 1
_frame_usec.append(Time.get_ticks_usec() - began)
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
_finish()
return true
return false
## How many distinct worlds this process's clients still reach (their own, the ground
## their prediction runs on, and islands made in the background but not yet taken
## up), and how many Simulations and grounds are alive in all. `extra_islands` counts
## live Simulations nothing here accounts for: a leak (#39).
func _live_worlds() -> Dictionary:
var worlds: Dictionary = {}
var grounds: Dictionary = {}
var built: int = 0
for client: Dictionary in _clients:
var game: ClientGame = client["game"]
if game.world != null:
worlds[game.world.get_instance_id()] = true
var ground: SimGround = game.prediction.get("_ground")
if ground != null:
grounds[ground.get_instance_id()] = true
var made: Simulation = game.get("_built")
if made != null and made != game.world:
built += 1
if game.building_world():
built += 1
var live: int = LiveCount.of(&"Simulation")
return {"worlds": worlds.size(), "prediction_grounds": grounds.size(), "built": built,
"live_simulations": live, "live_grounds": LiveCount.of(&"SimGround"), "extra_islands": maxi(0, live - worlds.size() - built)}
## Keeps track of a client's phases, and reports each match it played once it's over.
func _follow(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var link: NetClient = client["link"]
if link.state == NetClient.CLOSED and not client.has("closed_s"):
# Say at once when a connection closes, with what its relay last saw each way.
var relay: UdpRelay = client["relay"]
var now: int = Time.get_ticks_msec()
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
print(JSON.stringify({
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
}))
if game.world == null:
return
var phases: Array = client["phases"]
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
phases.append(game.match_phase)
if game.playing():
client["frames_playing"] = int(client["frames_playing"]) + 1
client["stats_match"] = game.match_index
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
elif int(client["stats_match"]) >= 0:
_report_match(client)
# Once a match is in play every client here has let go of the last match's world.
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
_memory_logged[game.match_index] = Time.get_ticks_msec()
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
elif game.match_phase == Match.PLAY and _memory_logged.has(game.match_index) and _memory_logged.has(-1 - game.match_index) == false \
and Time.get_ticks_msec() - int(_memory_logged[game.match_index]) > LATE_MEMORY_MSEC:
# A second look a minute later, for telling a leak from something let go late.
_memory_logged[-1 - game.match_index] = true
print(JSON.stringify({"event": "memory_late", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
func _report_match(client: Dictionary) -> void:
var stats: Array = client["last_stats"]
var record: Dictionary = {
"event": "client_match", "client": client["number"], "match": client["stats_match"],
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
}
(client["matches"] as Array).append(record)
print(JSON.stringify(record))
client["stats_match"] = -1
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
## world has caught up to exactly what the snapshot says the server had sent.
func _sample(client: Dictionary) -> void:
var game: ClientGame = client["game"]
var tick: int = game.receiver.sim_tick
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
# so it names more than one state; the server doesn't log it either.
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
return
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
client["skipped"] = int(client["skipped"]) + 1
return
client["samples"] = int(client["samples"]) + 1
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
_tamper = -1
game.world.props.health[0] -= 1
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
func _finish() -> void:
_done = true
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
if int(client["stats_match"]) >= 0:
_report_match(client)
var phase_names: Array[String] = []
for phase: int in client["phases"]:
phase_names.append(Match.PHASE_NAMES[phase])
print(JSON.stringify({
"event": "soak_client", "client": client["number"], "name": link.player_name,
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
"world_gaps": game.replica.gaps,
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
}))
for key: String in totals:
totals[key] = int(totals[key]) + int(relay.get(key))
link.leave()
relay.close()
# The input and snapshot callables hold the client: let go of them, or nothing frees.
game.sample_input = Callable()
game.on_snapshot = Callable()
var sorted: PackedInt64Array = _frame_usec.duplicate()
sorted.sort()
var summary: Dictionary = {
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
}
if not sorted.is_empty():
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
summary.merge(totals)
if _errors.count() > 0:
summary["first_error"] = _errors.first()
print(JSON.stringify(summary))
_clients.clear()
OS.remove_logger(_errors)
quit(1 if _errors.count() > 0 else 0)