N8: Keep connections serviced while islands are made, and stop servers gracefully

The soak's drops under loss came from stalls: the server's island (#79) and
each client process making islands on its main thread fed ENet second-long
round-trip samples, its retransmission timer grew, and two losses of one
reliable message in the busy drop that followed reached the 15 s timeout.

- ClientGame.build_in_background makes a new match's island on a
  WorkerThreadPool task and holds the packets that arrive meanwhile, then
  takes them in order. The networked client and the soak's clients use it;
  the soak's own one-island-a-frame holding is gone.
- NetServer.close(grace_msec) sends the shutdown notice reliably and
  services the socket while clients acknowledge it. The headless server
  waits up to 2 s. A notice sent once was lost under 2% loss, and that
  client timed out instead of hearing the server stop.
- Tests: background island holding and ordering, freeing mid-build, a
  graceful shutdown reaching four clients behind 50% loss (it fails with
  the old one-shot notice), and the scripted UDP match and client capture
  wait for the client's island the way a real 20 s lobby would.

The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches,
with 0 of 3,758 world hashes differing and every client told of the stop.
It still fails on client memory growth between matches in two of three
processes (17 and 20 MB), which is not yet explained, so #39 stays open.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-13 01:28:08 -05:00
co-authored by Claude Opus 5.5
parent ad0fd2695d
commit 09d663e647
9 changed files with 259 additions and 64 deletions
+101 -2
View File
@@ -44,6 +44,14 @@ var outbox: Array[PackedByteArray] = []
var net: NetClient = null
## Packets that made no sense, so far.
var bad_packets: int = 0
## When true, a new match's island is made on a WorkerThreadPool task instead of in
## receive(), and the packets that arrive meanwhile are held and then taken in order.
## A networked client sets it, so the connection keeps being serviced: a second-long
## stall feeds ENet huge round-trip samples, and its retransmission timer then grows
## until two losses of one message can time the connection out (N8). Tests that
## expect `world` the moment a notice arrives leave it off. `make_world` must be safe
## to call from another thread when it is on.
var build_in_background: bool = false
## Called after each snapshot has been taken in, with no arguments. Soak tests use it
## to compare the client's world with the server's at that moment.
var on_snapshot: Callable = Callable()
@@ -53,12 +61,35 @@ var _ack_due: bool = false
var _last_sim_tick: int = -1
var _last_yaw: int = 0
var _last_pitch: int = 0
## The background island build, if one is running: its task, the seed, a box the
## task writes the world into, and the packets held until it is done.
var _build_task: int = -1
var _build_seed: int = 0
var _build_box: Array = []
var _built: Simulation = null
var _held: Array[Array] = []
func attach(client: NetClient) -> void:
net = client
func _notification(what: int) -> void:
if what == NOTIFICATION_PREDELETE and _build_task >= 0:
WorkerThreadPool.wait_for_task_completion(_build_task)
_build_task = -1
## True while a new match's island is being made in the background.
func building_world() -> bool:
return _build_task >= 0
## How many packets are held while the island is made.
func held_count() -> int:
return _held.size()
## Services the attached NetClient and takes every packet that arrived.
func poll(now_msec: int = -1) -> void:
if net == null:
@@ -68,8 +99,71 @@ func poll(now_msec: int = -1) -> void:
receive(int(packet["channel"]), packet["bytes"])
## Takes one packet from the server.
## Takes one packet from the server. With build_in_background, a notice of a match on
## a new island starts making it, and it and everything after it wait until it's made.
func receive(channel: int, bytes: PackedByteArray) -> void:
_collect_world()
if _build_task >= 0 or not _held.is_empty():
_held.append([channel, bytes])
return
if build_in_background and _needs_new_island(channel, bytes):
_start_build(NetSnapshot.decode_match(bytes)["seed"])
_held.append([channel, bytes])
return
_receive_now(channel, bytes)
## Takes in the held packets once the background island is made. receive() and
## advance() call it; call it yourself to check without either.
func _collect_world() -> void:
if _build_task < 0 or not WorkerThreadPool.is_task_completed(_build_task):
return
WorkerThreadPool.wait_for_task_completion(_build_task)
_build_task = -1
_built = _build_box[0] as Simulation
_build_box = []
var held: Array[Array] = _held
_held = []
for index: int in held.size():
var packet: Array = held[index]
if _build_task >= 0:
# Another new island was asked for while taking these in: hold the rest.
_held.append_array(held.slice(index))
return
if build_in_background and _needs_new_island(int(packet[0]), packet[1]):
_start_build(NetSnapshot.decode_match(packet[1])["seed"])
_held.append_array(held.slice(index))
return
_receive_now(int(packet[0]), packet[1])
## True for a MATCH notice whose seed isn't the world's or the one just made.
func _needs_new_island(channel: int, bytes: PackedByteArray) -> bool:
if channel != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
return false
var notice: Dictionary = NetSnapshot.decode_match(bytes)
if notice.is_empty():
return false
var seed_value: int = int(notice["seed"])
if _built != null and _built.rng.match_seed == seed_value:
return false
return world == null or seed_value != world_seed
func _start_build(seed_value: int) -> void:
# The shared item table is filled here, so the worker never races to build it.
ItemCatalog.all()
_build_seed = seed_value
var box: Array = [null]
_build_box = box
_build_task = WorkerThreadPool.add_task(_build_into.bind(box, make_world, seed_value), false, "Tidewall client island")
static func _build_into(box: Array, maker: Callable, seed_value: int) -> void:
box[0] = maker.call(seed_value)
func _receive_now(channel: int, bytes: PackedByteArray) -> void:
if channel == NetProtocol.CHANNEL_CONTROL:
var type: int = bytes[0] if not bytes.is_empty() else 0
if type == NetWorld.MSG_WORLD:
@@ -101,6 +195,7 @@ func receive(channel: int, bytes: PackedByteArray) -> void:
## Runs the client for `delta` seconds: makes and predicts the commands that are due,
## and moves the drawing clocks on. Call once a frame.
func advance(delta: float) -> void:
_collect_world()
remote.advance(delta)
prediction.smooth(delta)
if prediction.active:
@@ -150,7 +245,11 @@ func _take_notice(notice: Dictionary) -> void:
func _make_world() -> void:
world = make_world.call(world_seed)
if _built != null and _built.rng.match_seed == world_seed:
world = _built
else:
world = make_world.call(world_seed)
_built = null
replica.begin(world)
+23 -3
View File
@@ -145,11 +145,23 @@ func kick(handle: int, reason: int = NetProtocol.REASON_KICKED) -> bool:
return true
## Stops listening. Everyone connected is told the server shut down.
func close() -> void:
## Stops listening. Everyone connected is told the server shut down. With
## `grace_msec`, the notice goes reliably and the socket is serviced for up to that
## long while clients acknowledge it; a notice sent only once is lost as often as any
## datagram, and that client is left to time out. Whoever is still connected after the
## grace gets the one-off notice.
func close(grace_msec: int = 0) -> void:
if _host == null:
return
for peer: ENetPacketPeer in _guests.keys() + _handle_of.keys() + _leaving.keys():
var peers: Array = _guests.keys() + _handle_of.keys() + _leaving.keys()
if grace_msec > 0:
for peer: ENetPacketPeer in peers:
if peer.is_active():
peer.peer_disconnect(NetProtocol.REASON_SHUTDOWN)
var until: int = Time.get_ticks_msec() + grace_msec
while Time.get_ticks_msec() < until and _any_active(peers):
_host.service(10)
for peer: ENetPacketPeer in peers:
if peer.is_active():
peer.peer_disconnect_now(NetProtocol.REASON_SHUTDOWN)
_host.flush()
@@ -218,6 +230,14 @@ func _gone(peer: ENetPacketPeer, data: int) -> void:
_handle_of.erase(peer)
func _any_active(peers: Array) -> bool:
for peer: ENetPacketPeer in peers:
# A peer that has finished disconnecting is let go of, and is no longer active.
if peer.is_active():
return true
return false
func _guests_from(address: String) -> int:
var count: int = 0
for peer: ENetPacketPeer in _guests: