N8: Keep connections serviced while islands are made, and stop servers gracefully

The soak's drops under loss came from stalls: the server's island (#79) and
each client process making islands on its main thread fed ENet second-long
round-trip samples, its retransmission timer grew, and two losses of one
reliable message in the busy drop that followed reached the 15 s timeout.

- ClientGame.build_in_background makes a new match's island on a
  WorkerThreadPool task and holds the packets that arrive meanwhile, then
  takes them in order. The networked client and the soak's clients use it;
  the soak's own one-island-a-frame holding is gone.
- NetServer.close(grace_msec) sends the shutdown notice reliably and
  services the socket while clients acknowledge it. The headless server
  waits up to 2 s. A notice sent once was lost under 2% loss, and that
  client timed out instead of hearing the server stop.
- Tests: background island holding and ordering, freeing mid-build, a
  graceful shutdown reaching four clients behind 50% loss (it fails with
  the old one-shot notice), and the scripted UDP match and client capture
  wait for the client's island the way a real 20 s lobby would.

The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches,
with 0 of 3,758 world hashes differing and every client told of the stop.
It still fails on client memory growth between matches in two of three
processes (17 and 20 MB), which is not yet explained, so #39 stays open.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 01:28:08 -05:00
co-authored by Claude Opus 5.5
parent ad0fd2695d
commit 09d663e647
9 changed files with 259 additions and 64 deletions
+2
View File
@@ -34,6 +34,8 @@ func _init() -> void:
func connect_to(host: String, port: int, player_name: String) -> Error:
link = NetClient.new()
game.attach(link)
# Make each new island off the main thread, so the connection keeps being serviced.
game.build_in_background = true
return link.connect_to(host, port, player_name)
+100 -1
View File
@@ -44,6 +44,14 @@ var outbox: Array[PackedByteArray] = []
var net: NetClient = null
## Packets that made no sense, so far.
var bad_packets: int = 0
## When true, a new match's island is made on a WorkerThreadPool task instead of in
## receive(), and the packets that arrive meanwhile are held and then taken in order.
## A networked client sets it, so the connection keeps being serviced: a second-long
## stall feeds ENet huge round-trip samples, and its retransmission timer then grows
## until two losses of one message can time the connection out (N8). Tests that
## expect `world` the moment a notice arrives leave it off. `make_world` must be safe
## to call from another thread when it is on.
var build_in_background: bool = false
## Called after each snapshot has been taken in, with no arguments. Soak tests use it
## to compare the client's world with the server's at that moment.
var on_snapshot: Callable = Callable()
@@ -53,12 +61,35 @@ var _ack_due: bool = false
var _last_sim_tick: int = -1
var _last_yaw: int = 0
var _last_pitch: int = 0
## The background island build, if one is running: its task, the seed, a box the
## task writes the world into, and the packets held until it is done.
var _build_task: int = -1
var _build_seed: int = 0
var _build_box: Array = []
var _built: Simulation = null
var _held: Array[Array] = []
func attach(client: NetClient) -> void:
net = client
func _notification(what: int) -> void:
if what == NOTIFICATION_PREDELETE and _build_task >= 0:
WorkerThreadPool.wait_for_task_completion(_build_task)
_build_task = -1
## True while a new match's island is being made in the background.
func building_world() -> bool:
return _build_task >= 0
## How many packets are held while the island is made.
func held_count() -> int:
return _held.size()
## Services the attached NetClient and takes every packet that arrived.
func poll(now_msec: int = -1) -> void:
if net == null:
@@ -68,8 +99,71 @@ func poll(now_msec: int = -1) -> void:
receive(int(packet["channel"]), packet["bytes"])
## Takes one packet from the server.
## Takes one packet from the server. With build_in_background, a notice of a match on
## a new island starts making it, and it and everything after it wait until it's made.
func receive(channel: int, bytes: PackedByteArray) -> void:
_collect_world()
if _build_task >= 0 or not _held.is_empty():
_held.append([channel, bytes])
return
if build_in_background and _needs_new_island(channel, bytes):
_start_build(NetSnapshot.decode_match(bytes)["seed"])
_held.append([channel, bytes])
return
_receive_now(channel, bytes)
## Takes in the held packets once the background island is made. receive() and
## advance() call it; call it yourself to check without either.
func _collect_world() -> void:
if _build_task < 0 or not WorkerThreadPool.is_task_completed(_build_task):
return
WorkerThreadPool.wait_for_task_completion(_build_task)
_build_task = -1
_built = _build_box[0] as Simulation
_build_box = []
var held: Array[Array] = _held
_held = []
for index: int in held.size():
var packet: Array = held[index]
if _build_task >= 0:
# Another new island was asked for while taking these in: hold the rest.
_held.append_array(held.slice(index))
return
if build_in_background and _needs_new_island(int(packet[0]), packet[1]):
_start_build(NetSnapshot.decode_match(packet[1])["seed"])
_held.append_array(held.slice(index))
return
_receive_now(int(packet[0]), packet[1])
## True for a MATCH notice whose seed isn't the world's or the one just made.
func _needs_new_island(channel: int, bytes: PackedByteArray) -> bool:
if channel != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
return false
var notice: Dictionary = NetSnapshot.decode_match(bytes)
if notice.is_empty():
return false
var seed_value: int = int(notice["seed"])
if _built != null and _built.rng.match_seed == seed_value:
return false
return world == null or seed_value != world_seed
func _start_build(seed_value: int) -> void:
# The shared item table is filled here, so the worker never races to build it.
ItemCatalog.all()
_build_seed = seed_value
var box: Array = [null]
_build_box = box
_build_task = WorkerThreadPool.add_task(_build_into.bind(box, make_world, seed_value), false, "Tidewall client island")
static func _build_into(box: Array, maker: Callable, seed_value: int) -> void:
box[0] = maker.call(seed_value)
func _receive_now(channel: int, bytes: PackedByteArray) -> void:
if channel == NetProtocol.CHANNEL_CONTROL:
var type: int = bytes[0] if not bytes.is_empty() else 0
if type == NetWorld.MSG_WORLD:
@@ -101,6 +195,7 @@ func receive(channel: int, bytes: PackedByteArray) -> void:
## Runs the client for `delta` seconds: makes and predicts the commands that are due,
## and moves the drawing clocks on. Call once a frame.
func advance(delta: float) -> void:
_collect_world()
remote.advance(delta)
prediction.smooth(delta)
if prediction.active:
@@ -150,7 +245,11 @@ func _take_notice(notice: Dictionary) -> void:
func _make_world() -> void:
if _built != null and _built.rng.match_seed == world_seed:
world = _built
else:
world = make_world.call(world_seed)
_built = null
replica.begin(world)
+23 -3
View File
@@ -145,11 +145,23 @@ func kick(handle: int, reason: int = NetProtocol.REASON_KICKED) -> bool:
return true
## Stops listening. Everyone connected is told the server shut down.
func close() -> void:
## Stops listening. Everyone connected is told the server shut down. With
## `grace_msec`, the notice goes reliably and the socket is serviced for up to that
## long while clients acknowledge it; a notice sent only once is lost as often as any
## datagram, and that client is left to time out. Whoever is still connected after the
## grace gets the one-off notice.
func close(grace_msec: int = 0) -> void:
if _host == null:
return
for peer: ENetPacketPeer in _guests.keys() + _handle_of.keys() + _leaving.keys():
var peers: Array = _guests.keys() + _handle_of.keys() + _leaving.keys()
if grace_msec > 0:
for peer: ENetPacketPeer in peers:
if peer.is_active():
peer.peer_disconnect(NetProtocol.REASON_SHUTDOWN)
var until: int = Time.get_ticks_msec() + grace_msec
while Time.get_ticks_msec() < until and _any_active(peers):
_host.service(10)
for peer: ENetPacketPeer in peers:
if peer.is_active():
peer.peer_disconnect_now(NetProtocol.REASON_SHUTDOWN)
_host.flush()
@@ -218,6 +230,14 @@ func _gone(peer: ENetPacketPeer, data: int) -> void:
_handle_of.erase(peer)
func _any_active(peers: Array) -> bool:
for peer: ENetPacketPeer in peers:
# A peer that has finished disconnecting is let go of, and is no longer active.
if peer.is_active():
return true
return false
func _guests_from(address: String) -> int:
var count: int = 0
for peer: ENetPacketPeer in _guests:
+4 -1
View File
@@ -9,6 +9,8 @@ extends Node
## Catch-up limit from docs/ARCHITECTURE.md.
const MAX_TICKS_PER_FRAME: int = 5
## How long a stopping server waits for clients to acknowledge that it is stopping.
const SHUTDOWN_GRACE_MSEC: int = 2000
## Ticks per frame when unthrottled. Keeps the engine's main loop responsive.
const UNTHROTTLED_BATCH: int = 250
## Engine frame cap in real-time mode, so an idle server doesn't spin a core.
@@ -209,7 +211,8 @@ func _finish(requested_code: int) -> void:
_finished = true
var code: int = requested_code
if game != null:
game.net.close()
# Up to two seconds for clients to acknowledge that the server is stopping.
game.net.close(SHUTDOWN_GRACE_MSEC)
if beacon != null:
beacon.close()
if health != null:
+6
View File
@@ -43,6 +43,12 @@ func _ready() -> void:
var since: int = -1
for _round: int in LIMIT_ROUNDS:
_game.poll()
if _session.game.building_world():
# The client makes its island in the background. A real lobby is far
# longer than that takes; this one is 6 ticks, so the server waits.
_session.pump(DT)
OS.delay_msec(5)
continue
_loop.run_tick()
_game.after_tick()
_session.pump(DT)
+6
View File
@@ -118,6 +118,12 @@ func test_a_scripted_client_plays_a_whole_match_on_a_real_server() -> void:
# agree however fast this machine is.
while Time.get_ticks_msec() - started < 240000 and not session.phases.has(Match.RESULTS):
game.poll()
if session.game.building_world():
# The island is made in the background. A real lobby lasts 20 s, far
# longer than that takes; this one is 10 ticks, so the server waits.
session.pump(DT)
OS.delay_msec(5)
continue
loop.run_tick()
game.after_tick()
session.pump(DT)
+56
View File
@@ -333,3 +333,59 @@ func test_handshakes_held_open_from_one_address_are_capped() -> void:
assert_eq(server.guest_count(), 2)
for event: Dictionary in _events("refused"):
assert_eq(event["reason"], NetProtocol.REASON_BUSY)
## N8: a server stopping with a grace sends its notice reliably, so clients behind a
## link that loses half its datagrams still hear it instead of timing out.
func test_a_graceful_shutdown_reaches_clients_on_a_lossy_link() -> void:
var relays: Array[UdpRelay] = []
var lossy: Array[NetClient] = []
for index: int in 4:
var relay: UdpRelay = UdpRelay.new(0.0, 0.0, 0.0, 11 + index)
assert_eq(relay.open(0, HOST, server.port()), OK)
relays.append(relay)
var client: NetClient = NetClient.new()
assert_eq(client.connect_to(HOST, relay.port(), "Lossy%d" % index), OK)
lossy.append(client)
var joined: Callable = func() -> bool:
for relay: UdpRelay in relays:
relay.poll()
for client: NetClient in lossy:
if client.state != NetClient.JOINED:
return false
return true
for _round: int in 2000:
server.poll()
for client: NetClient in lossy:
client.poll()
if joined.call():
break
OS.delay_msec(2)
assert_true(joined.call(), "every client joined")
for relay: UdpRelay in relays:
relay.loss = 0.5
# The clients go on being serviced in another thread while the server closes.
var stop: Array = [false]
var worker: Thread = Thread.new()
worker.start(func() -> void:
while not stop[0]:
for relay: UdpRelay in relays:
relay.poll()
for client: NetClient in lossy:
client.poll()
OS.delay_msec(2))
server.close(3000)
var started: int = Time.get_ticks_msec()
var all_closed: bool = false
while Time.get_ticks_msec() - started < 3000 and not all_closed:
OS.delay_msec(20)
all_closed = true
for client: NetClient in lossy:
all_closed = all_closed and client.state == NetClient.CLOSED
stop[0] = true
worker.wait_to_finish()
for client: NetClient in lossy:
assert_eq(client.state, NetClient.CLOSED)
assert_eq(client.close_reason, NetProtocol.REASON_SHUTDOWN, "%s was told the server stopped" % client.player_name)
for relay: UdpRelay in relays:
relay.close()
+54
View File
@@ -151,3 +151,57 @@ func test_a_new_match_starts_everything_again() -> void:
assert_false(client.remote.started)
assert_eq(worlds, [first_seed, first_seed + 1])
assert_eq(client.match_index, 1)
## N8: with build_in_background the island is made on a worker thread, and the packets
## that arrive meanwhile are held, then taken in order once it's made.
func test_a_background_island_holds_packets_and_takes_them_in_order() -> void:
var worlds: Array[int] = []
var client: ClientGame = ClientGame.new()
client.build_in_background = true
client.make_world = func(match_seed: int) -> Simulation:
# Slow enough that the packets below certainly arrive while it runs.
OS.delay_msec(300)
worlds.append(match_seed)
return Simulation.new(match_seed, SimGround.flat())
loop.match_flow.rules.min_players = 5
assert_true(loop.match_flow.join(7))
game.adopt(7)
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
assert_true(client.building_world(), "the notice starts the island in the background")
assert_null(client.world, "and nothing waits for it")
client.receive(NetProtocol.CHANNEL_CONTROL, game.world_notice())
loop.run_tick()
game.after_tick()
client.receive(NetProtocol.CHANNEL_STATE, game.snapshot_for(7))
assert_eq(client.held_count(), 3, "the notice, the world and the snapshot are held")
assert_eq(client.receiver.malformed, 0)
var started: int = Time.get_ticks_msec()
while client.building_world() and Time.get_ticks_msec() - started < 5000:
client.advance(DT)
OS.delay_msec(5)
assert_false(client.building_world())
assert_eq(client.held_count(), 0, "everything held was taken in")
assert_eq(worlds, [loop.match_flow.current_seed()], "one island, for the notice's seed")
assert_not_null(client.world)
assert_eq(client.world_seed, loop.match_flow.current_seed())
assert_eq(client.match_index, 0)
assert_true(client.replica.applied >= 1, "the WORLD message went on the new island")
assert_eq(client.outbox.size(), 1, "and the snapshot was answered")
assert_eq(client.bad_packets, 0)
# The same seed again doesn't make another island.
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
assert_false(client.building_world())
assert_eq(worlds.size(), 1)
func test_a_client_freed_while_its_island_is_made_waits_for_it() -> void:
var client: ClientGame = ClientGame.new()
client.build_in_background = true
client.make_world = func(match_seed: int) -> Simulation:
OS.delay_msec(100)
return Simulation.new(match_seed, SimGround.flat())
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
assert_true(client.building_world())
client = null
assert_true(true, "freeing it with a task running raises no error")
+7 -58
View File
@@ -21,11 +21,9 @@
## {"event":"soak_process", ...}: frame times, totals and errors, last.
## It stops when every client's connection has closed, or after --seconds.
##
## Making a match's island takes most of a second. A real player does it once, but a
## dozen clients here would all do it in the same frame and starve every connection of
## service for many seconds. So a MATCH notice that brings a new match is held back,
## with everything after it for that client, until the driver has made that client's
## world, and the driver makes at most one world a frame.
## Making a match's island takes most of a second. Each client makes it on a worker
## thread (ClientGame.build_in_background), as the real client does, so the frame loop
## keeps servicing every connection while islands are made.
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
extends SceneTree
@@ -78,7 +76,6 @@ func _initialize() -> void:
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
"held": [], "notice": "", "ready_world": null, "ready_seed": 0,
}
var game: ClientGame = client["game"]
var player: SoakPlayer = client["player"]
@@ -87,12 +84,7 @@ func _initialize() -> void:
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
return player.command(tick, game.prediction.player, game.world)
game.on_snapshot = _sample.bind(client)
game.make_world = func(match_seed: int) -> Simulation:
var ready: Simulation = client["ready_world"]
client["ready_world"] = null
if ready != null and int(client["ready_seed"]) == match_seed:
return ready
return Simulation.new(match_seed)
game.build_in_background = true
game.attach(client["link"])
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
_clients.append(client)
@@ -103,17 +95,14 @@ func _process(delta: float) -> bool:
return true
var began: int = Time.get_ticks_usec()
var open: int = 0
var may_make_world: bool = true
for client: Dictionary in _clients:
var relay: UdpRelay = client["relay"]
var link: NetClient = client["link"]
var game: ClientGame = client["game"]
relay.poll()
link.poll()
_take_packets(client, link.packets)
if may_make_world and not (client["held"] as Array).is_empty():
may_make_world = false
_make_world(client)
for packet: Dictionary in link.packets:
game.receive(int(packet["channel"]), packet["bytes"])
game.advance(delta)
relay.poll()
_follow(client)
@@ -126,44 +115,6 @@ func _process(delta: float) -> bool:
return false
## Passes packets to the client's game, holding back a MATCH notice for a new match,
## and everything after it, until its world has been made.
func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void:
var game: ClientGame = client["game"]
var held: Array = client["held"]
for packet: Dictionary in packets:
if held.is_empty() and not _brings_new_match(client, packet):
game.receive(int(packet["channel"]), packet["bytes"])
else:
held.append(packet)
## True for a MATCH notice of a match (index and seed) the client hasn't been told of.
func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool:
var bytes: PackedByteArray = packet["bytes"]
if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
return false
var notice: Dictionary = NetSnapshot.decode_match(bytes)
if notice.is_empty():
return false
var key: String = "%d:%d" % [notice["index"], notice["seed"]]
if key == String(client["notice"]):
return false
client["notice"] = key
client["ready_seed"] = int(notice["seed"])
return true
## Makes the world for the held notice, then passes on everything held.
func _make_world(client: Dictionary) -> void:
var game: ClientGame = client["game"]
client["ready_world"] = Simulation.new(int(client["ready_seed"]))
var held: Array = client["held"]
client["held"] = []
for packet: Dictionary in held:
game.receive(int(packet["channel"]), packet["bytes"])
## Keeps track of a client's phases, and reports each match it played once it's over.
func _follow(client: Dictionary) -> void:
var game: ClientGame = client["game"]
@@ -176,7 +127,7 @@ func _follow(client: Dictionary) -> void:
print(JSON.stringify({
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
"held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(),
"held": game.held_count(), "relay_in_flight": relay.in_flight(),
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
}))
@@ -256,8 +207,6 @@ func _finish() -> void:
# The input and snapshot callables hold the client: let go of them, or nothing frees.
game.sample_input = Callable()
game.on_snapshot = Callable()
game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed)
client["ready_world"] = null
var sorted: PackedInt64Array = _frame_usec.duplicate()
sorted.sort()
var summary: Dictionary = {