N8: Keep connections serviced while islands are made, and stop servers gracefully

The soak's drops under loss came from stalls: the server's island (#79) and
each client process making islands on its main thread fed ENet second-long
round-trip samples, its retransmission timer grew, and two losses of one
reliable message in the busy drop that followed reached the 15 s timeout.

- ClientGame.build_in_background makes a new match's island on a
  WorkerThreadPool task and holds the packets that arrive meanwhile, then
  takes them in order. The networked client and the soak's clients use it;
  the soak's own one-island-a-frame holding is gone.
- NetServer.close(grace_msec) sends the shutdown notice reliably and
  services the socket while clients acknowledge it. The headless server
  waits up to 2 s. A notice sent once was lost under 2% loss, and that
  client timed out instead of hearing the server stop.
- Tests: background island holding and ordering, freeing mid-build, a
  graceful shutdown reaching four clients behind 50% loss (it fails with
  the old one-shot notice), and the scripted UDP match and client capture
  wait for the client's island the way a real 20 s lobby would.

The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches,
with 0 of 3,758 world hashes differing and every client told of the stop.
It still fails on client memory growth between matches in two of three
processes (17 and 20 MB), which is not yet explained, so #39 stays open.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 01:28:08 -05:00
co-authored by Claude Opus 5.5
parent ad0fd2695d
commit 09d663e647
9 changed files with 259 additions and 64 deletions
+7 -58
View File
@@ -21,11 +21,9 @@
## {"event":"soak_process", ...}: frame times, totals and errors, last.
## It stops when every client's connection has closed, or after --seconds.
##
## Making a match's island takes most of a second. A real player does it once, but a
## dozen clients here would all do it in the same frame and starve every connection of
## service for many seconds. So a MATCH notice that brings a new match is held back,
## with everything after it for that client, until the driver has made that client's
## world, and the driver makes at most one world a frame.
## Making a match's island takes most of a second. Each client makes it on a worker
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
## keeps servicing every connection while islands are made.
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
extends SceneTree
@@ -78,7 +76,6 @@ func _initialize() -> void:
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
"held": [], "notice": "", "ready_world": null, "ready_seed": 0,
}
var game: ClientGame = client["game"]
var player: SoakPlayer = client["player"]
@@ -87,12 +84,7 @@ func _initialize() -> void:
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
return player.command(tick, game.prediction.player, game.world)
game.on_snapshot = _sample.bind(client)
game.make_world = func(match_seed: int) -> Simulation:
var ready: Simulation = client["ready_world"]
client["ready_world"] = null
if ready != null and int(client["ready_seed"]) == match_seed:
return ready
return Simulation.new(match_seed)
game.build_in_background = true
game.attach(client["link"])
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
_clients.append(client)
@@ -103,17 +95,14 @@ func _process(delta: float) -> bool:
return true
var began: int = Time.get_ticks_usec()
var open: int = 0
var may_make_world: bool = true
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
relay.poll()
link.poll()
_take_packets(client, link.packets)
if may_make_world and not (client["held"] as Array).is_empty():
may_make_world = false
_make_world(client)
for packet: Dictionary in link.packets:
game.receive(int(packet["channel"]), packet["bytes"])
game.advance(delta)
relay.poll()
_follow(client)
@@ -126,44 +115,6 @@ func _process(delta: float) -> bool:
return false
## Passes packets to the client's game, holding back a MATCH notice for a new match,
## and everything after it, until its world has been made.
func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void:
var game: ClientGame = client["game"]
var held: Array = client["held"]
for packet: Dictionary in packets:
if held.is_empty() and not _brings_new_match(client, packet):
game.receive(int(packet["channel"]), packet["bytes"])
else:
held.append(packet)
## True for a MATCH notice of a match (index and seed) the client hasn't been told of.
func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool:
var bytes: PackedByteArray = packet["bytes"]
if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
return false
var notice: Dictionary = NetSnapshot.decode_match(bytes)
if notice.is_empty():
return false
var key: String = "%d:%d" % [notice["index"], notice["seed"]]
if key == String(client["notice"]):
return false
client["notice"] = key
client["ready_seed"] = int(notice["seed"])
return true
## Makes the world for the held notice, then passes on everything held.
func _make_world(client: Dictionary) -> void:
var game: ClientGame = client["game"]
client["ready_world"] = Simulation.new(int(client["ready_seed"]))
var held: Array = client["held"]
client["held"] = []
for packet: Dictionary in held:
game.receive(int(packet["channel"]), packet["bytes"])
## Keeps track of a client's phases, and reports each match it played once it's over.
func _follow(client: Dictionary) -> void:
var game: ClientGame = client["game"]
@@ -176,7 +127,7 @@ func _follow(client: Dictionary) -> void:
print(JSON.stringify({
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
"held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(),
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
}))
@@ -256,8 +207,6 @@ func _finish() -> void:
# The input and snapshot callables hold the client: let go of them, or nothing frees.
game.sample_input = Callable()
game.on_snapshot = Callable()
game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed)
client["ready_world"] = null
var sorted: PackedInt64Array = _frame_usec.duplicate()
sorted.sort()
var summary: Dictionary = {