N8: 24-client network soak, and the server fixes it found

make soak runs a real server and 24 headless clients in three processes for
two whole matches in real time. Each client reaches the server through its own
UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs
unchanged over a bad link. SoakPlayer drives each client from what a client
knows: it drops, glides, heads for high ground, harvests and tries to build.
tools/soak_report.gd checks world hashes against the server's, errors,
unreadable packets, every client playing every match with its connection
intact, memory between matches, and the p99 tick with the network's share.
make soak-selfcheck proves a drifted client world is caught.

Server changes: --matches=N, --world-hash-every=N, tick timing that includes
after_tick, and memory in match and progress lines.

What the soak found and fixed:
- Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the
  12 ms budget, against a 2.8 ms simulation. Encoded records are now shared
  between clients for the tick, unchanged records skip the field diff, and
  clients are spread over the snapshot interval by handle. p99 is 7.3 ms.
- ENet peers had a timeout limit of 0, which gives up on a peer as soon as any
  reliable message has waited the minimum timeout. Now ENet's default, 32.

Not fixed: one or two clients still lose their connection about 15 s into the
second match's drop in each run (see the issue), so the soak still fails and
N8 stays open.

Refs #39

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-12 21:56:01 -05:00
co-authored by Claude Opus 5.5
parent 65b1fe015c
commit 05c7568ec1
25 changed files with 1135 additions and 39 deletions
+27 -4
View File
@@ -26,6 +26,10 @@ var _errors: ErrorCounter = ErrorCounter.new()
var _accumulator: float = 0.0
var _started_usec: int = 0
var _finished: bool = false
var _matches_ended: int = 0
## When listening: microseconds each tick took including the network's share (after_tick:
## world messages and snapshots), which ServerLoop's own timing leaves out.
var _net_tick_usec: PackedInt64Array = PackedInt64Array()
func _init(server_config: ServerConfig) -> void:
@@ -91,10 +95,10 @@ func _process(delta: float) -> void:
# Too far behind to catch up: drop the backlog rather than spiral.
_accumulator = minf(_accumulator, 1.0 / Tuning.TICK_RATE_HZ)
for _index: int in due:
if config.ticks > 0 and loop.ticks >= config.ticks:
if _done():
break
_tick()
if config.ticks > 0 and loop.ticks >= config.ticks:
if _done():
_finish(0)
@@ -104,6 +108,11 @@ func _exit_tree() -> void:
_finish(0)
## True once the run has done what it was asked: its ticks, or its matches.
func _done() -> bool:
return (config.ticks > 0 and loop.ticks >= config.ticks) or (config.matches > 0 and _matches_ended >= config.matches)
func _open_beacon() -> void:
if not config.lan and config.lists.is_empty():
return
@@ -131,19 +140,26 @@ func _open_beacon() -> void:
func _tick() -> void:
if config.fault == "script-error" and loop.ticks == FAULT_TICK:
_deliberate_script_error()
var began: int = Time.get_ticks_usec()
loop.run_tick()
if game != null:
game.after_tick()
_net_tick_usec.append(Time.get_ticks_usec() - began)
if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
_log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim)})
for event: Dictionary in loop.match_flow.events:
if event["type"] in ["phase", "match_started", "match_ended"]:
_log("match", event)
var fields: Dictionary = event.duplicate()
fields["mem_bytes"] = OS.get_static_memory_usage()
_log("match", fields)
elif event["type"] == "placed":
_log("eliminated", event)
if event["type"] == "match_ended":
_log("results", {"match": event["match"], "summary": loop.match_flow.summary()})
_matches_ended += 1
if config.log_every > 0 and loop.ticks % config.log_every == 0:
var stats: Dictionary = loop.timing_stats()
_log("progress", {"tick": loop.ticks, "p95_us": stats["p95_us"], "max_us": stats["max_us"]})
_log("progress", {"tick": loop.ticks, "p95_us": stats["p95_us"], "max_us": stats["max_us"], "mem_bytes": OS.get_static_memory_usage()})
func _finish(requested_code: int) -> void:
@@ -157,6 +173,13 @@ func _finish(requested_code: int) -> void:
var stats: Dictionary = loop.timing_stats()
stats["state_hash"] = loop.sim.state_hash()
stats["entities"] = loop.sim.entities.count()
if not _net_tick_usec.is_empty():
var sorted: PackedInt64Array = _net_tick_usec.duplicate()
sorted.sort()
stats["with_net_p50_us"] = ServerLoop._percentile(sorted, 0.50)
stats["with_net_p99_us"] = ServerLoop._percentile(sorted, 0.99)
stats["with_net_max_us"] = sorted[sorted.size() - 1]
stats["mem_bytes"] = OS.get_static_memory_usage()
_log("stats", stats)
if code == 0 and _errors.count() > 0:
code = 1
+11 -4
View File
@@ -138,18 +138,25 @@ func after_tick() -> void:
if not bytes.is_empty():
own_changes[handle] = bytes
net.send(handle, NetProtocol.CHANNEL_CONTROL, bytes, true)
if not snapshot_due():
return
for handle: int in net.handles():
if sender.has_client(handle):
if sender.has_client(handle) and snapshot_due_for(handle):
net.send(handle, NetProtocol.CHANNEL_STATE, snapshot_for(handle), false)
## True on the ticks a snapshot goes out.
## True on the ticks a snapshot goes out to an owner that carries packets for a single
## client itself (LocalServer and tests).
func snapshot_due() -> bool:
return loop.ticks % Tuning.NET_SNAPSHOT_INTERVAL_TICKS == 0
## True on the ticks a networked client's snapshot goes out. Clients are spread over
## the ticks of the interval by handle, so the cost of building snapshots, the largest
## share of a busy server's tick, is split between them instead of landing on one.
func snapshot_due_for(handle: int) -> bool:
var interval: int = Tuning.NET_SNAPSHOT_INTERVAL_TICKS
return (loop.ticks + posmod(handle, interval)) % interval == 0
## The snapshot for a participant as things stand: what they may see, and their own
## block while they are playing.
func snapshot_for(handle: int) -> PackedByteArray:
+18 -1
View File
@@ -2,7 +2,7 @@
##
## tools/godot.sh -- --server [--listen] [--port=N] [--seed=N] [--players=N] [--bots=N]
## [--bot-difficulty=mixed|easy|normal|hard] [--ticks=N]
## [--unthrottled] [--log-every=N]
## [--unthrottled] [--log-every=N] [--matches=N] [--world-hash-every=N]
## [--name=NAME] [--list=HOST[:PORT]]... [--no-lan]
##
## With `--listen` the server takes network players on UDP `--port`, waits in the lobby
@@ -38,6 +38,12 @@ var bots: int = 0
var bot_difficulty: int = -1
## Ticks to run before exiting. 0 runs until the process is stopped.
var ticks: int = 0
## Stops after this many matches have ended. 0 doesn't stop for matches.
var matches: int = 0
## On the simulation ticks from each multiple of this many up to the snapshot interval
## (never tick 0, which lasts the whole lobby), logs NetWorld.world_hash() of the match's
## world, so a soak test can compare it with its clients' copies. 0 turns it off.
var world_hash_every: int = 0
## Runs ticks back to back instead of at the real tick rate. For tests and smoke runs.
var unthrottled: bool = false
## Logs a progress line every this many ticks. 0 turns progress lines off.
@@ -81,6 +87,7 @@ func to_dictionary() -> Dictionary:
"bots": bots,
"bot_difficulty": "mixed" if bot_difficulty < 0 else String(Tuning.BOT_DIFFICULTIES[bot_difficulty]["name"]),
"ticks": ticks,
"matches": matches,
"unthrottled": unthrottled,
"name": server_name,
"lan": lan and listen,
@@ -124,6 +131,12 @@ func _parse_one(arg: String) -> void:
"--log-every":
if _check_int(key, value):
log_every = value.to_int()
"--matches":
if _check_int(key, value):
matches = value.to_int()
"--world-hash-every":
if _check_int(key, value):
world_hash_every = value.to_int()
"--bot-difficulty":
var found: bool = value == "mixed"
for index: int in Tuning.BOT_DIFFICULTIES.size():
@@ -195,6 +208,10 @@ func _validate() -> void:
errors.append("--ticks can't be negative")
if log_every < 0:
errors.append("--log-every can't be negative")
if matches < 0 or world_hash_every < 0:
errors.append("--matches and --world-hash-every can't be negative")
if world_hash_every > 0 and not listen:
errors.append("--world-hash-every is for a server started with --listen")
if not listen and (not lists.is_empty() or not lan):
errors.append("--list and --no-lan are for a server started with --listen")
if fault not in FAULTS: