The soak's client memory growth (17 to 20 MB, about five islands, in some processes) came from SoakPlayer keeping a strong reference to the last world it played on. It let go only when its client next asked it for a command on the new world, so finished islands stayed alive across a match change. It now keeps only the island's seed. LiveCount counts live Simulations and SimGrounds, which are made on worker threads too, so it is thread-safe. The soak clients log the counts, and the soak report fails if, a minute into a match, any island is alive that isn't a client's world or one being made. That makes a leak name itself instead of showing up as bytes. Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
257 lines
12 KiB
GDScript
257 lines
12 KiB
GDScript
## Runs several headless network clients in one process for the soak test (make soak).
|
|
## Each client reaches the server through its own UdpRelay, which delays, jitters and
|
|
## loses datagrams both ways, and is played by a SoakPlayer.
|
|
##
|
|
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
|
|
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
|
|
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
|
|
##
|
|
## --tamper=N knocks a point off one prop's health in client N's world just before its
|
|
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
|
|
## client whose world has drifted is caught.
|
|
##
|
|
## Prints JSON lines:
|
|
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
|
|
## the snapshots where the server logs its own (--world-hash-every on the server, the
|
|
## same N as --hash-every here), when the client's world has caught up exactly.
|
|
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
|
|
## for a match it played, once the match is over for it.
|
|
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
|
|
## {"event":"soak_client", ...}: one per client at the end.
|
|
## {"event":"soak_process", ...}: frame times, totals and errors, last.
|
|
## It stops when every client's connection has closed, or after --seconds.
|
|
##
|
|
## Making a match's island takes most of a second. Each client makes it on a worker
|
|
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
|
|
## keeps servicing every connection while islands are made.
|
|
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
|
|
extends SceneTree
|
|
|
|
const FPS: int = 60
|
|
const LATE_MEMORY_MSEC: int = 60000
|
|
|
|
var _clients: Array[Dictionary] = []
|
|
var _errors: ErrorCounter = ErrorCounter.new()
|
|
var _hash_every: int = 300
|
|
var _deadline_msec: int = 0
|
|
var _frame_usec: PackedInt64Array = PackedInt64Array()
|
|
var _memory_logged: Dictionary = {}
|
|
var _started_msec: int = 0
|
|
var _done: bool = false
|
|
var _tamper: int = -1
|
|
|
|
|
|
func _initialize() -> void:
|
|
OS.add_logger(_errors)
|
|
var options: Dictionary = {
|
|
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
|
|
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
|
|
}
|
|
for arg: String in OS.get_cmdline_user_args():
|
|
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
|
if parts.size() != 2 or not options.has(parts[0]):
|
|
printerr("soak_clients: unknown argument '%s'" % arg)
|
|
quit(2)
|
|
return
|
|
options[parts[0]] = parts[1]
|
|
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
|
|
if target.is_empty():
|
|
printerr("soak_clients: --server=HOST:PORT is needed")
|
|
quit(2)
|
|
return
|
|
Engine.max_fps = FPS
|
|
_hash_every = int(options["hash-every"])
|
|
_tamper = int(options["tamper"])
|
|
_started_msec = Time.get_ticks_msec()
|
|
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
|
|
var first: int = int(options["first"])
|
|
for index: int in int(options["clients"]):
|
|
var number: int = first + index
|
|
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
|
|
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
|
|
if error != OK:
|
|
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
|
|
quit(2)
|
|
return
|
|
var client: Dictionary = {
|
|
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
|
|
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
|
|
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
|
|
}
|
|
var game: ClientGame = client["game"]
|
|
var player: SoakPlayer = client["player"]
|
|
game.sample_input = func(tick: int) -> PlayerCommand:
|
|
if game.world == null:
|
|
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
|
|
return player.command(tick, game.prediction.player, game.world)
|
|
game.on_snapshot = _sample.bind(client)
|
|
game.build_in_background = true
|
|
game.attach(client["link"])
|
|
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
|
|
_clients.append(client)
|
|
|
|
|
|
func _process(delta: float) -> bool:
|
|
if _done:
|
|
return true
|
|
var began: int = Time.get_ticks_usec()
|
|
var open: int = 0
|
|
for client: Dictionary in _clients:
|
|
var relay: UdpRelay = client["relay"]
|
|
var link: NetClient = client["link"]
|
|
var game: ClientGame = client["game"]
|
|
relay.poll()
|
|
link.poll()
|
|
for packet: Dictionary in link.packets:
|
|
game.receive(int(packet["channel"]), packet["bytes"])
|
|
game.advance(delta)
|
|
relay.poll()
|
|
_follow(client)
|
|
if link.state != NetClient.CLOSED:
|
|
open += 1
|
|
_frame_usec.append(Time.get_ticks_usec() - began)
|
|
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
|
|
_finish()
|
|
return true
|
|
return false
|
|
|
|
|
|
## How many distinct worlds this process's clients still reach (their own, the ground
|
|
## their prediction runs on, and islands made in the background but not yet taken
|
|
## up), and how many Simulations and grounds are alive in all. `extra_islands` counts
|
|
## live Simulations nothing here accounts for: a leak (#39).
|
|
func _live_worlds() -> Dictionary:
|
|
var worlds: Dictionary = {}
|
|
var grounds: Dictionary = {}
|
|
var built: int = 0
|
|
for client: Dictionary in _clients:
|
|
var game: ClientGame = client["game"]
|
|
if game.world != null:
|
|
worlds[game.world.get_instance_id()] = true
|
|
var ground: SimGround = game.prediction.get("_ground")
|
|
if ground != null:
|
|
grounds[ground.get_instance_id()] = true
|
|
var made: Simulation = game.get("_built")
|
|
if made != null and made != game.world:
|
|
built += 1
|
|
if game.building_world():
|
|
built += 1
|
|
var live: int = LiveCount.of(&"Simulation")
|
|
return {"worlds": worlds.size(), "prediction_grounds": grounds.size(), "built": built,
|
|
"live_simulations": live, "live_grounds": LiveCount.of(&"SimGround"), "extra_islands": maxi(0, live - worlds.size() - built)}
|
|
|
|
|
|
## Keeps track of a client's phases, and reports each match it played once it's over.
|
|
func _follow(client: Dictionary) -> void:
|
|
var game: ClientGame = client["game"]
|
|
var link: NetClient = client["link"]
|
|
if link.state == NetClient.CLOSED and not client.has("closed_s"):
|
|
# Say at once when a connection closes, with what its relay last saw each way.
|
|
var relay: UdpRelay = client["relay"]
|
|
var now: int = Time.get_ticks_msec()
|
|
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
|
|
print(JSON.stringify({
|
|
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
|
|
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
|
|
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
|
|
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
|
|
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
|
|
}))
|
|
if game.world == null:
|
|
return
|
|
var phases: Array = client["phases"]
|
|
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
|
|
phases.append(game.match_phase)
|
|
if game.playing():
|
|
client["frames_playing"] = int(client["frames_playing"]) + 1
|
|
client["stats_match"] = game.match_index
|
|
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
|
|
elif int(client["stats_match"]) >= 0:
|
|
_report_match(client)
|
|
# Once a match is in play every client here has let go of the last match's world.
|
|
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
|
|
_memory_logged[game.match_index] = Time.get_ticks_msec()
|
|
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
|
|
elif game.match_phase == Match.PLAY and _memory_logged.has(game.match_index) and _memory_logged.has(-1 - game.match_index) == false \
|
|
and Time.get_ticks_msec() - int(_memory_logged[game.match_index]) > LATE_MEMORY_MSEC:
|
|
# A second look a minute later, for telling a leak from something let go late.
|
|
_memory_logged[-1 - game.match_index] = true
|
|
print(JSON.stringify({"event": "memory_late", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()}))
|
|
|
|
|
|
func _report_match(client: Dictionary) -> void:
|
|
var stats: Array = client["last_stats"]
|
|
var record: Dictionary = {
|
|
"event": "client_match", "client": client["number"], "match": client["stats_match"],
|
|
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
|
|
}
|
|
(client["matches"] as Array).append(record)
|
|
print(JSON.stringify(record))
|
|
client["stats_match"] = -1
|
|
|
|
|
|
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
|
|
## world has caught up to exactly what the snapshot says the server had sent.
|
|
func _sample(client: Dictionary) -> void:
|
|
var game: ClientGame = client["game"]
|
|
var tick: int = game.receiver.sim_tick
|
|
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
|
|
# so it names more than one state; the server doesn't log it either.
|
|
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
|
return
|
|
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
|
|
client["skipped"] = int(client["skipped"]) + 1
|
|
return
|
|
client["samples"] = int(client["samples"]) + 1
|
|
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
|
|
_tamper = -1
|
|
game.world.props.health[0] -= 1
|
|
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
|
|
|
|
|
|
func _finish() -> void:
|
|
_done = true
|
|
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
|
|
for client: Dictionary in _clients:
|
|
var relay: UdpRelay = client["relay"]
|
|
var link: NetClient = client["link"]
|
|
var game: ClientGame = client["game"]
|
|
if int(client["stats_match"]) >= 0:
|
|
_report_match(client)
|
|
var phase_names: Array[String] = []
|
|
for phase: int in client["phases"]:
|
|
phase_names.append(Match.PHASE_NAMES[phase])
|
|
print(JSON.stringify({
|
|
"event": "soak_client", "client": client["number"], "name": link.player_name,
|
|
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
|
|
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
|
|
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
|
|
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
|
|
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
|
|
"world_gaps": game.replica.gaps,
|
|
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
|
|
}))
|
|
for key: String in totals:
|
|
totals[key] = int(totals[key]) + int(relay.get(key))
|
|
link.leave()
|
|
relay.close()
|
|
# The input and snapshot callables hold the client: let go of them, or nothing frees.
|
|
game.sample_input = Callable()
|
|
game.on_snapshot = Callable()
|
|
var sorted: PackedInt64Array = _frame_usec.duplicate()
|
|
sorted.sort()
|
|
var summary: Dictionary = {
|
|
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
|
|
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
|
|
}
|
|
if not sorted.is_empty():
|
|
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
|
|
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
|
|
summary.merge(totals)
|
|
if _errors.count() > 0:
|
|
summary["first_error"] = _errors.first()
|
|
print(JSON.stringify(summary))
|
|
_clients.clear()
|
|
OS.remove_logger(_errors)
|
|
quit(1 if _errors.count() > 0 else 0)
|