diff --git a/src/app/client_session.gd b/src/app/client_session.gd index e661f7f..e642aa3 100644 --- a/src/app/client_session.gd +++ b/src/app/client_session.gd @@ -34,6 +34,8 @@ func _init() -> void: func connect_to(host: String, port: int, player_name: String) -> Error: link = NetClient.new() game.attach(link) + # Make each new island off the main thread, so the connection keeps being serviced. + game.build_in_background = true return link.connect_to(host, port, player_name) diff --git a/src/net/client_game.gd b/src/net/client_game.gd index da7bcac..56bb4ef 100644 --- a/src/net/client_game.gd +++ b/src/net/client_game.gd @@ -44,6 +44,14 @@ var outbox: Array[PackedByteArray] = [] var net: NetClient = null ## Packets that made no sense, so far. var bad_packets: int = 0 +## When true, a new match's island is made on a WorkerThreadPool task instead of in +## receive(), and the packets that arrive meanwhile are held and then taken in order. +## A networked client sets it, so the connection keeps being serviced: a second-long +## stall feeds ENet huge round-trip samples, and its retransmission timer then grows +## until two losses of one message can time the connection out (N8). Tests that +## expect `world` the moment a notice arrives leave it off. `make_world` must be safe +## to call from another thread when it is on. +var build_in_background: bool = false ## Called after each snapshot has been taken in, with no arguments. Soak tests use it ## to compare the client's world with the server's at that moment. var on_snapshot: Callable = Callable() @@ -53,12 +61,35 @@ var _ack_due: bool = false var _last_sim_tick: int = -1 var _last_yaw: int = 0 var _last_pitch: int = 0 +## The background island build, if one is running: its task, the seed, a box the +## task writes the world into, and the packets held until it is done. +var _build_task: int = -1 +var _build_seed: int = 0 +var _build_box: Array = [] +var _built: Simulation = null +var _held: Array[Array] = [] func attach(client: NetClient) -> void: net = client +func _notification(what: int) -> void: + if what == NOTIFICATION_PREDELETE and _build_task >= 0: + WorkerThreadPool.wait_for_task_completion(_build_task) + _build_task = -1 + + +## True while a new match's island is being made in the background. +func building_world() -> bool: + return _build_task >= 0 + + +## How many packets are held while the island is made. +func held_count() -> int: + return _held.size() + + ## Services the attached NetClient and takes every packet that arrived. func poll(now_msec: int = -1) -> void: if net == null: @@ -68,8 +99,71 @@ func poll(now_msec: int = -1) -> void: receive(int(packet["channel"]), packet["bytes"]) -## Takes one packet from the server. +## Takes one packet from the server. With build_in_background, a notice of a match on +## a new island starts making it, and it and everything after it wait until it's made. func receive(channel: int, bytes: PackedByteArray) -> void: + _collect_world() + if _build_task >= 0 or not _held.is_empty(): + _held.append([channel, bytes]) + return + if build_in_background and _needs_new_island(channel, bytes): + _start_build(NetSnapshot.decode_match(bytes)["seed"]) + _held.append([channel, bytes]) + return + _receive_now(channel, bytes) + + +## Takes in the held packets once the background island is made. receive() and +## advance() call it; call it yourself to check without either. +func _collect_world() -> void: + if _build_task < 0 or not WorkerThreadPool.is_task_completed(_build_task): + return + WorkerThreadPool.wait_for_task_completion(_build_task) + _build_task = -1 + _built = _build_box[0] as Simulation + _build_box = [] + var held: Array[Array] = _held + _held = [] + for index: int in held.size(): + var packet: Array = held[index] + if _build_task >= 0: + # Another new island was asked for while taking these in: hold the rest. + _held.append_array(held.slice(index)) + return + if build_in_background and _needs_new_island(int(packet[0]), packet[1]): + _start_build(NetSnapshot.decode_match(packet[1])["seed"]) + _held.append_array(held.slice(index)) + return + _receive_now(int(packet[0]), packet[1]) + + +## True for a MATCH notice whose seed isn't the world's or the one just made. +func _needs_new_island(channel: int, bytes: PackedByteArray) -> bool: + if channel != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH: + return false + var notice: Dictionary = NetSnapshot.decode_match(bytes) + if notice.is_empty(): + return false + var seed_value: int = int(notice["seed"]) + if _built != null and _built.rng.match_seed == seed_value: + return false + return world == null or seed_value != world_seed + + +func _start_build(seed_value: int) -> void: + # The shared item table is filled here, so the worker never races to build it. + ItemCatalog.all() + _build_seed = seed_value + var box: Array = [null] + _build_box = box + _build_task = WorkerThreadPool.add_task(_build_into.bind(box, make_world, seed_value), false, "Tidewall client island") + + +static func _build_into(box: Array, maker: Callable, seed_value: int) -> void: + box[0] = maker.call(seed_value) + + +func _receive_now(channel: int, bytes: PackedByteArray) -> void: if channel == NetProtocol.CHANNEL_CONTROL: var type: int = bytes[0] if not bytes.is_empty() else 0 if type == NetWorld.MSG_WORLD: @@ -101,6 +195,7 @@ func receive(channel: int, bytes: PackedByteArray) -> void: ## Runs the client for `delta` seconds: makes and predicts the commands that are due, ## and moves the drawing clocks on. Call once a frame. func advance(delta: float) -> void: + _collect_world() remote.advance(delta) prediction.smooth(delta) if prediction.active: @@ -150,7 +245,11 @@ func _take_notice(notice: Dictionary) -> void: func _make_world() -> void: - world = make_world.call(world_seed) + if _built != null and _built.rng.match_seed == world_seed: + world = _built + else: + world = make_world.call(world_seed) + _built = null replica.begin(world) diff --git a/src/net/net_server.gd b/src/net/net_server.gd index 1893f5a..6b47f68 100644 --- a/src/net/net_server.gd +++ b/src/net/net_server.gd @@ -145,11 +145,23 @@ func kick(handle: int, reason: int = NetProtocol.REASON_KICKED) -> bool: return true -## Stops listening. Everyone connected is told the server shut down. -func close() -> void: +## Stops listening. Everyone connected is told the server shut down. With +## `grace_msec`, the notice goes reliably and the socket is serviced for up to that +## long while clients acknowledge it; a notice sent only once is lost as often as any +## datagram, and that client is left to time out. Whoever is still connected after the +## grace gets the one-off notice. +func close(grace_msec: int = 0) -> void: if _host == null: return - for peer: ENetPacketPeer in _guests.keys() + _handle_of.keys() + _leaving.keys(): + var peers: Array = _guests.keys() + _handle_of.keys() + _leaving.keys() + if grace_msec > 0: + for peer: ENetPacketPeer in peers: + if peer.is_active(): + peer.peer_disconnect(NetProtocol.REASON_SHUTDOWN) + var until: int = Time.get_ticks_msec() + grace_msec + while Time.get_ticks_msec() < until and _any_active(peers): + _host.service(10) + for peer: ENetPacketPeer in peers: if peer.is_active(): peer.peer_disconnect_now(NetProtocol.REASON_SHUTDOWN) _host.flush() @@ -218,6 +230,14 @@ func _gone(peer: ENetPacketPeer, data: int) -> void: _handle_of.erase(peer) +func _any_active(peers: Array) -> bool: + for peer: ENetPacketPeer in peers: + # A peer that has finished disconnecting is let go of, and is no longer active. + if peer.is_active(): + return true + return false + + func _guests_from(address: String) -> int: var count: int = 0 for peer: ENetPacketPeer in _guests: diff --git a/src/server/headless_server.gd b/src/server/headless_server.gd index 7cd3e26..aa25d6d 100644 --- a/src/server/headless_server.gd +++ b/src/server/headless_server.gd @@ -9,6 +9,8 @@ extends Node ## Catch-up limit from docs/ARCHITECTURE.md. const MAX_TICKS_PER_FRAME: int = 5 +## How long a stopping server waits for clients to acknowledge that it is stopping. +const SHUTDOWN_GRACE_MSEC: int = 2000 ## Ticks per frame when unthrottled. Keeps the engine's main loop responsive. const UNTHROTTLED_BATCH: int = 250 ## Engine frame cap in real-time mode, so an idle server doesn't spin a core. @@ -209,7 +211,8 @@ func _finish(requested_code: int) -> void: _finished = true var code: int = requested_code if game != null: - game.net.close() + # Up to two seconds for clients to acknowledge that the server is stopping. + game.net.close(SHUTDOWN_GRACE_MSEC) if beacon != null: beacon.close() if health != null: diff --git a/test/capture/client_scene.gd b/test/capture/client_scene.gd index 4b914c1..4c698a4 100644 --- a/test/capture/client_scene.gd +++ b/test/capture/client_scene.gd @@ -43,6 +43,12 @@ func _ready() -> void: var since: int = -1 for _round: int in LIMIT_ROUNDS: _game.poll() + if _session.game.building_world(): + # The client makes its island in the background. A real lobby is far + # longer than that takes; this one is 6 ticks, so the server waits. + _session.pump(DT) + OS.delay_msec(5) + continue _loop.run_tick() _game.after_tick() _session.pump(DT) diff --git a/test/client/test_client_session.gd b/test/client/test_client_session.gd index 0ad93f9..45e8c78 100644 --- a/test/client/test_client_session.gd +++ b/test/client/test_client_session.gd @@ -118,6 +118,12 @@ func test_a_scripted_client_plays_a_whole_match_on_a_real_server() -> void: # agree however fast this machine is. while Time.get_ticks_msec() - started < 240000 and not session.phases.has(Match.RESULTS): game.poll() + if session.game.building_world(): + # The island is made in the background. A real lobby lasts 20 s, far + # longer than that takes; this one is 10 ticks, so the server waits. + session.pump(DT) + OS.delay_msec(5) + continue loop.run_tick() game.after_tick() session.pump(DT) diff --git a/test/net/test_net_handshake.gd b/test/net/test_net_handshake.gd index 5f2301d..06276c3 100644 --- a/test/net/test_net_handshake.gd +++ b/test/net/test_net_handshake.gd @@ -333,3 +333,59 @@ func test_handshakes_held_open_from_one_address_are_capped() -> void: assert_eq(server.guest_count(), 2) for event: Dictionary in _events("refused"): assert_eq(event["reason"], NetProtocol.REASON_BUSY) + + +## N8: a server stopping with a grace sends its notice reliably, so clients behind a +## link that loses half its datagrams still hear it instead of timing out. +func test_a_graceful_shutdown_reaches_clients_on_a_lossy_link() -> void: + var relays: Array[UdpRelay] = [] + var lossy: Array[NetClient] = [] + for index: int in 4: + var relay: UdpRelay = UdpRelay.new(0.0, 0.0, 0.0, 11 + index) + assert_eq(relay.open(0, HOST, server.port()), OK) + relays.append(relay) + var client: NetClient = NetClient.new() + assert_eq(client.connect_to(HOST, relay.port(), "Lossy%d" % index), OK) + lossy.append(client) + var joined: Callable = func() -> bool: + for relay: UdpRelay in relays: + relay.poll() + for client: NetClient in lossy: + if client.state != NetClient.JOINED: + return false + return true + for _round: int in 2000: + server.poll() + for client: NetClient in lossy: + client.poll() + if joined.call(): + break + OS.delay_msec(2) + assert_true(joined.call(), "every client joined") + for relay: UdpRelay in relays: + relay.loss = 0.5 + # The clients go on being serviced in another thread while the server closes. + var stop: Array = [false] + var worker: Thread = Thread.new() + worker.start(func() -> void: + while not stop[0]: + for relay: UdpRelay in relays: + relay.poll() + for client: NetClient in lossy: + client.poll() + OS.delay_msec(2)) + server.close(3000) + var started: int = Time.get_ticks_msec() + var all_closed: bool = false + while Time.get_ticks_msec() - started < 3000 and not all_closed: + OS.delay_msec(20) + all_closed = true + for client: NetClient in lossy: + all_closed = all_closed and client.state == NetClient.CLOSED + stop[0] = true + worker.wait_to_finish() + for client: NetClient in lossy: + assert_eq(client.state, NetClient.CLOSED) + assert_eq(client.close_reason, NetProtocol.REASON_SHUTDOWN, "%s was told the server stopped" % client.player_name) + for relay: UdpRelay in relays: + relay.close() diff --git a/test/server/test_client_game.gd b/test/server/test_client_game.gd index b142554..9313c95 100644 --- a/test/server/test_client_game.gd +++ b/test/server/test_client_game.gd @@ -151,3 +151,57 @@ func test_a_new_match_starts_everything_again() -> void: assert_false(client.remote.started) assert_eq(worlds, [first_seed, first_seed + 1]) assert_eq(client.match_index, 1) + + +## N8: with build_in_background the island is made on a worker thread, and the packets +## that arrive meanwhile are held, then taken in order once it's made. +func test_a_background_island_holds_packets_and_takes_them_in_order() -> void: + var worlds: Array[int] = [] + var client: ClientGame = ClientGame.new() + client.build_in_background = true + client.make_world = func(match_seed: int) -> Simulation: + # Slow enough that the packets below certainly arrive while it runs. + OS.delay_msec(300) + worlds.append(match_seed) + return Simulation.new(match_seed, SimGround.flat()) + loop.match_flow.rules.min_players = 5 + assert_true(loop.match_flow.join(7)) + game.adopt(7) + client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice()) + assert_true(client.building_world(), "the notice starts the island in the background") + assert_null(client.world, "and nothing waits for it") + client.receive(NetProtocol.CHANNEL_CONTROL, game.world_notice()) + loop.run_tick() + game.after_tick() + client.receive(NetProtocol.CHANNEL_STATE, game.snapshot_for(7)) + assert_eq(client.held_count(), 3, "the notice, the world and the snapshot are held") + assert_eq(client.receiver.malformed, 0) + var started: int = Time.get_ticks_msec() + while client.building_world() and Time.get_ticks_msec() - started < 5000: + client.advance(DT) + OS.delay_msec(5) + assert_false(client.building_world()) + assert_eq(client.held_count(), 0, "everything held was taken in") + assert_eq(worlds, [loop.match_flow.current_seed()], "one island, for the notice's seed") + assert_not_null(client.world) + assert_eq(client.world_seed, loop.match_flow.current_seed()) + assert_eq(client.match_index, 0) + assert_true(client.replica.applied >= 1, "the WORLD message went on the new island") + assert_eq(client.outbox.size(), 1, "and the snapshot was answered") + assert_eq(client.bad_packets, 0) + # The same seed again doesn't make another island. + client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice()) + assert_false(client.building_world()) + assert_eq(worlds.size(), 1) + + +func test_a_client_freed_while_its_island_is_made_waits_for_it() -> void: + var client: ClientGame = ClientGame.new() + client.build_in_background = true + client.make_world = func(match_seed: int) -> Simulation: + OS.delay_msec(100) + return Simulation.new(match_seed, SimGround.flat()) + client.receive(NetProtocol.CHANNEL_CONTROL, game.match_notice()) + assert_true(client.building_world()) + client = null + assert_true(true, "freeing it with a task running raises no error") diff --git a/tools/soak_clients.gd b/tools/soak_clients.gd index 8dbcb46..8f49cbb 100644 --- a/tools/soak_clients.gd +++ b/tools/soak_clients.gd @@ -21,11 +21,9 @@ ## {"event":"soak_process", ...}: frame times, totals and errors, last. ## It stops when every client's connection has closed, or after --seconds. ## -## Making a match's island takes most of a second. A real player does it once, but a -## dozen clients here would all do it in the same frame and starve every connection of -## service for many seconds. So a MATCH notice that brings a new match is held back, -## with everything after it for that client, until the driver has made that client's -## world, and the driver makes at most one world a frame. +## Making a match's island takes most of a second. Each client makes it on a worker +## thread (ClientGame.build_in_background), as the real client does, so the frame loop +## keeps servicing every connection while islands are made. ## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line. extends SceneTree @@ -78,7 +76,6 @@ func _initialize() -> void: "number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(), "player": SoakPlayer.new(number), "phases": [], "frames_playing": 0, "samples": 0, "skipped": 0, "matches": [], "stats_match": -1, - "held": [], "notice": "", "ready_world": null, "ready_seed": 0, } var game: ClientGame = client["game"] var player: SoakPlayer = client["player"] @@ -87,12 +84,7 @@ func _initialize() -> void: return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0) return player.command(tick, game.prediction.player, game.world) game.on_snapshot = _sample.bind(client) - game.make_world = func(match_seed: int) -> Simulation: - var ready: Simulation = client["ready_world"] - client["ready_world"] = null - if ready != null and int(client["ready_seed"]) == match_seed: - return ready - return Simulation.new(match_seed) + game.build_in_background = true game.attach(client["link"]) (client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number) _clients.append(client) @@ -103,17 +95,14 @@ func _process(delta: float) -> bool: return true var began: int = Time.get_ticks_usec() var open: int = 0 - var may_make_world: bool = true for client: Dictionary in _clients: var relay: UdpRelay = client["relay"] var link: NetClient = client["link"] var game: ClientGame = client["game"] relay.poll() link.poll() - _take_packets(client, link.packets) - if may_make_world and not (client["held"] as Array).is_empty(): - may_make_world = false - _make_world(client) + for packet: Dictionary in link.packets: + game.receive(int(packet["channel"]), packet["bytes"]) game.advance(delta) relay.poll() _follow(client) @@ -126,44 +115,6 @@ func _process(delta: float) -> bool: return false -## Passes packets to the client's game, holding back a MATCH notice for a new match, -## and everything after it, until its world has been made. -func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void: - var game: ClientGame = client["game"] - var held: Array = client["held"] - for packet: Dictionary in packets: - if held.is_empty() and not _brings_new_match(client, packet): - game.receive(int(packet["channel"]), packet["bytes"]) - else: - held.append(packet) - - -## True for a MATCH notice of a match (index and seed) the client hasn't been told of. -func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool: - var bytes: PackedByteArray = packet["bytes"] - if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH: - return false - var notice: Dictionary = NetSnapshot.decode_match(bytes) - if notice.is_empty(): - return false - var key: String = "%d:%d" % [notice["index"], notice["seed"]] - if key == String(client["notice"]): - return false - client["notice"] = key - client["ready_seed"] = int(notice["seed"]) - return true - - -## Makes the world for the held notice, then passes on everything held. -func _make_world(client: Dictionary) -> void: - var game: ClientGame = client["game"] - client["ready_world"] = Simulation.new(int(client["ready_seed"])) - var held: Array = client["held"] - client["held"] = [] - for packet: Dictionary in held: - game.receive(int(packet["channel"]), packet["bytes"]) - - ## Keeps track of a client's phases, and reports each match it played once it's over. func _follow(client: Dictionary) -> void: var game: ClientGame = client["game"] @@ -176,7 +127,7 @@ func _follow(client: Dictionary) -> void: print(JSON.stringify({ "event": "client_closed", "client": client["number"], "reason": link.close_reason, "at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index, - "held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(), + "held": game.held_count(), "relay_in_flight": relay.in_flight(), "since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1, "since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1, })) @@ -256,8 +207,6 @@ func _finish() -> void: # The input and snapshot callables hold the client: let go of them, or nothing frees. game.sample_input = Callable() game.on_snapshot = Callable() - game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed) - client["ready_world"] = null var sorted: PackedInt64Array = _frame_usec.duplicate() sorted.sort() var summary: Dictionary = {