N8: Keep connections serviced while islands are made, and stop servers gracefully
The soak's drops under loss came from stalls: the server's island (#79) and each client process making islands on its main thread fed ENet second-long round-trip samples, its retransmission timer grew, and two losses of one reliable message in the busy drop that followed reached the 15 s timeout. - ClientGame.build_in_background makes a new match's island on a WorkerThreadPool task and holds the packets that arrive meanwhile, then takes them in order. The networked client and the soak's clients use it; the soak's own one-island-a-frame holding is gone. - NetServer.close(grace_msec) sends the shutdown notice reliably and services the socket while clients acknowledge it. The headless server waits up to 2 s. A notice sent once was lost under 2% loss, and that client timed out instead of hearing the server stop. - Tests: background island holding and ordering, freeing mid-build, a graceful shutdown reaching four clients behind 50% loss (it fails with the old one-shot notice), and the scripted UDP match and client capture wait for the client's island the way a real 20 s lobby would. The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches, with 0 of 3,758 world hashes differing and every client told of the stop. It still fails on client memory growth between matches in two of three processes (17 and 20 MB), which is not yet explained, so #39 stays open. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -43,6 +43,12 @@ func _ready() -> void:
|
||||
var since: int = -1
|
||||
for _round: int in LIMIT_ROUNDS:
|
||||
_game.poll()
|
||||
if _session.game.building_world():
|
||||
# The client makes its island in the background. A real lobby is far
|
||||
# longer than that takes; this one is 6 ticks, so the server waits.
|
||||
_session.pump(DT)
|
||||
OS.delay_msec(5)
|
||||
continue
|
||||
_loop.run_tick()
|
||||
_game.after_tick()
|
||||
_session.pump(DT)
|
||||
|
||||
@@ -118,6 +118,12 @@ func test_a_scripted_client_plays_a_whole_match_on_a_real_server() -> void:
|
||||
# agree however fast this machine is.
|
||||
while Time.get_ticks_msec() - started < 240000 and not session.phases.has(Match.RESULTS):
|
||||
game.poll()
|
||||
if session.game.building_world():
|
||||
# The island is made in the background. A real lobby lasts 20 s, far
|
||||
# longer than that takes; this one is 10 ticks, so the server waits.
|
||||
session.pump(DT)
|
||||
OS.delay_msec(5)
|
||||
continue
|
||||
loop.run_tick()
|
||||
game.after_tick()
|
||||
session.pump(DT)
|
||||
|
||||
@@ -333,3 +333,59 @@ func test_handshakes_held_open_from_one_address_are_capped() -> void:
|
||||
assert_eq(server.guest_count(), 2)
|
||||
for event: Dictionary in _events("refused"):
|
||||
assert_eq(event["reason"], NetProtocol.REASON_BUSY)
|
||||
|
||||
|
||||
## N8: a server stopping with a grace sends its notice reliably, so clients behind a
|
||||
## link that loses half its datagrams still hear it instead of timing out.
|
||||
func test_a_graceful_shutdown_reaches_clients_on_a_lossy_link() -> void:
|
||||
var relays: Array[UdpRelay] = []
|
||||
var lossy: Array[NetClient] = []
|
||||
for index: int in 4:
|
||||
var relay: UdpRelay = UdpRelay.new(0.0, 0.0, 0.0, 11 + index)
|
||||
assert_eq(relay.open(0, HOST, server.port()), OK)
|
||||
relays.append(relay)
|
||||
var client: NetClient = NetClient.new()
|
||||
assert_eq(client.connect_to(HOST, relay.port(), "Lossy%d" % index), OK)
|
||||
lossy.append(client)
|
||||
var joined: Callable = func() -> bool:
|
||||
for relay: UdpRelay in relays:
|
||||
relay.poll()
|
||||
for client: NetClient in lossy:
|
||||
if client.state != NetClient.JOINED:
|
||||
return false
|
||||
return true
|
||||
for _round: int in 2000:
|
||||
server.poll()
|
||||
for client: NetClient in lossy:
|
||||
client.poll()
|
||||
if joined.call():
|
||||
break
|
||||
OS.delay_msec(2)
|
||||
assert_true(joined.call(), "every client joined")
|
||||
for relay: UdpRelay in relays:
|
||||
relay.loss = 0.5
|
||||
# The clients go on being serviced in another thread while the server closes.
|
||||
var stop: Array = [false]
|
||||
var worker: Thread = Thread.new()
|
||||
worker.start(func() -> void:
|
||||
while not stop[0]:
|
||||
for relay: UdpRelay in relays:
|
||||
relay.poll()
|
||||
for client: NetClient in lossy:
|
||||
client.poll()
|
||||
OS.delay_msec(2))
|
||||
server.close(3000)
|
||||
var started: int = Time.get_ticks_msec()
|
||||
var all_closed: bool = false
|
||||
while Time.get_ticks_msec() - started < 3000 and not all_closed:
|
||||
OS.delay_msec(20)
|
||||
all_closed = true
|
||||
for client: NetClient in lossy:
|
||||
all_closed = all_closed and client.state == NetClient.CLOSED
|
||||
stop[0] = true
|
||||
worker.wait_to_finish()
|
||||
for client: NetClient in lossy:
|
||||
assert_eq(client.state, NetClient.CLOSED)
|
||||
assert_eq(client.close_reason, NetProtocol.REASON_SHUTDOWN, "%s was told the server stopped" % client.player_name)
|
||||
for relay: UdpRelay in relays:
|
||||
relay.close()
|
||||
|
||||
@@ -151,3 +151,57 @@ func test_a_new_match_starts_everything_again() -> void:
|
||||
assert_false(client.remote.started)
|
||||
assert_eq(worlds, [first_seed, first_seed + 1])
|
||||
assert_eq(client.match_index, 1)
|
||||
|
||||
|
||||
## N8: with build_in_background the island is made on a worker thread, and the packets
|
||||
## that arrive meanwhile are held, then taken in order once it's made.
|
||||
func test_a_background_island_holds_packets_and_takes_them_in_order() -> void:
|
||||
var worlds: Array[int] = []
|
||||
var client: ClientGame = ClientGame.new()
|
||||
client.build_in_background = true
|
||||
client.make_world = func(match_seed: int) -> Simulation:
|
||||
# Slow enough that the packets below certainly arrive while it runs.
|
||||
OS.delay_msec(300)
|
||||
worlds.append(match_seed)
|
||||
return Simulation.new(match_seed, SimGround.flat())
|
||||
loop.match_flow.rules.min_players = 5
|
||||
assert_true(loop.match_flow.join(7))
|
||||
game.adopt(7)
|
||||
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
|
||||
assert_true(client.building_world(), "the notice starts the island in the background")
|
||||
assert_null(client.world, "and nothing waits for it")
|
||||
client.receive(NetProtocol.CHANNEL_CONTROL, game.world_notice())
|
||||
loop.run_tick()
|
||||
game.after_tick()
|
||||
client.receive(NetProtocol.CHANNEL_STATE, game.snapshot_for(7))
|
||||
assert_eq(client.held_count(), 3, "the notice, the world and the snapshot are held")
|
||||
assert_eq(client.receiver.malformed, 0)
|
||||
var started: int = Time.get_ticks_msec()
|
||||
while client.building_world() and Time.get_ticks_msec() - started < 5000:
|
||||
client.advance(DT)
|
||||
OS.delay_msec(5)
|
||||
assert_false(client.building_world())
|
||||
assert_eq(client.held_count(), 0, "everything held was taken in")
|
||||
assert_eq(worlds, [loop.match_flow.current_seed()], "one island, for the notice's seed")
|
||||
assert_not_null(client.world)
|
||||
assert_eq(client.world_seed, loop.match_flow.current_seed())
|
||||
assert_eq(client.match_index, 0)
|
||||
assert_true(client.replica.applied >= 1, "the WORLD message went on the new island")
|
||||
assert_eq(client.outbox.size(), 1, "and the snapshot was answered")
|
||||
assert_eq(client.bad_packets, 0)
|
||||
# The same seed again doesn't make another island.
|
||||
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
|
||||
assert_false(client.building_world())
|
||||
assert_eq(worlds.size(), 1)
|
||||
|
||||
|
||||
func test_a_client_freed_while_its_island_is_made_waits_for_it() -> void:
|
||||
var client: ClientGame = ClientGame.new()
|
||||
client.build_in_background = true
|
||||
client.make_world = func(match_seed: int) -> Simulation:
|
||||
OS.delay_msec(100)
|
||||
return Simulation.new(match_seed, SimGround.flat())
|
||||
client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice())
|
||||
assert_true(client.building_world())
|
||||
client = null
|
||||
assert_true(true, "freeing it with a task running raises no error")
|
||||
|
||||
Reference in New Issue
Block a user