The soak's drops under loss came from stalls: the server's island (#79) and each client process making islands on its main thread fed ENet second-long round-trip samples, its retransmission timer grew, and two losses of one reliable message in the busy drop that followed reached the 15 s timeout. - ClientGame.build_in_background makes a new match's island on a WorkerThreadPool task and holds the packets that arrive meanwhile, then takes them in order. The networked client and the soak's clients use it; the soak's own one-island-a-frame holding is gone. - NetServer.close(grace_msec) sends the shutdown notice reliably and services the socket while clients acknowledge it. The headless server waits up to 2 s. A notice sent once was lost under 2% loss, and that client timed out instead of hearing the server stop. - Tests: background island holding and ordering, freeing mid-build, a graceful shutdown reaching four clients behind 50% loss (it fails with the old one-shot notice), and the scripted UDP match and client capture wait for the client's island the way a real 20 s lobby would. The soak at 2% loss and +/-20 ms now keeps all 24 clients for both matches, with 0 of 3,758 world hashes differing and every client told of the stop. It still fails on client memory growth between matches in two of three processes (17 and 20 MB), which is not yet explained, so #39 stays open. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
392 lines
16 KiB
GDScript
392 lines
16 KiB
GDScript
extends TestCase
|
|
|
|
## Issue N1: a real NetServer and NetClients over loopback UDP. Handshake, refusals,
|
|
## clean leaves, timeouts, and hostile or broken peers.
|
|
|
|
const HOST: String = "127.0.0.1"
|
|
|
|
var server: NetServer
|
|
var clients: Array[NetClient] = []
|
|
## Every server event since the test began.
|
|
var seen: Array[Dictionary] = []
|
|
var raw_hosts: Array[ENetConnection] = []
|
|
## Game packets the server handed to its owner since the test began.
|
|
var handed: int = 0
|
|
|
|
|
|
func before_each() -> void:
|
|
server = NetServer.new()
|
|
server.match_seed = 4242
|
|
clients.clear()
|
|
seen.clear()
|
|
handed = 0
|
|
raw_hosts.clear()
|
|
assert_eq(server.listen(0, HOST), OK)
|
|
assert_true(server.port() > 0)
|
|
|
|
|
|
func after_each() -> void:
|
|
for client: NetClient in clients:
|
|
client.leave()
|
|
for host: ENetConnection in raw_hosts:
|
|
host.destroy()
|
|
server.close()
|
|
|
|
|
|
func _client(wanted: String) -> NetClient:
|
|
var client: NetClient = NetClient.new()
|
|
clients.append(client)
|
|
assert_eq(client.connect_to(HOST, server.port(), wanted), OK)
|
|
return client
|
|
|
|
|
|
## Polls everything until `done` is true or `limit_msec` passes. Returns `done`'s answer.
|
|
func _pump(done: Callable, limit_msec: int = 4000) -> bool:
|
|
var started: int = Time.get_ticks_msec()
|
|
while Time.get_ticks_msec() - started < limit_msec:
|
|
server.poll()
|
|
seen.append_array(server.events)
|
|
handed += server.packets.size()
|
|
for client: NetClient in clients:
|
|
client.poll()
|
|
for host: ENetConnection in raw_hosts:
|
|
_service_raw(host)
|
|
if done.call():
|
|
return true
|
|
OS.delay_msec(2)
|
|
return bool(done.call())
|
|
|
|
|
|
func _events(type: String) -> Array[Dictionary]:
|
|
var found: Array[Dictionary] = []
|
|
for event: Dictionary in seen:
|
|
if event["type"] == type:
|
|
found.append(event)
|
|
return found
|
|
|
|
|
|
# A bare ENet peer, for sending what a real client never would.
|
|
|
|
var _raw_state: Dictionary = {}
|
|
|
|
|
|
func _raw() -> ENetPacketPeer:
|
|
var host: ENetConnection = ENetConnection.new()
|
|
assert_eq(host.create_host(1, NetProtocol.CHANNEL_COUNT), OK)
|
|
var peer: ENetPacketPeer = host.connect_to_host(HOST, server.port(), NetProtocol.CHANNEL_COUNT)
|
|
raw_hosts.append(host)
|
|
_raw_state[host] = {"connected": false, "disconnect_data": -1, "packets": []}
|
|
assert_true(_pump(func() -> bool: return _raw_state[host]["connected"]), "the raw peer connected")
|
|
return peer
|
|
|
|
|
|
func _service_raw(host: ENetConnection) -> void:
|
|
while true:
|
|
var event: Array = host.service(0)
|
|
if event[0] == ENetConnection.EVENT_NONE or event[0] == ENetConnection.EVENT_ERROR:
|
|
return
|
|
var state: Dictionary = _raw_state[host]
|
|
if event[0] == ENetConnection.EVENT_CONNECT:
|
|
state["connected"] = true
|
|
elif event[0] == ENetConnection.EVENT_DISCONNECT:
|
|
state["disconnect_data"] = int(event[2])
|
|
elif event[0] == ENetConnection.EVENT_RECEIVE:
|
|
state["packets"].append((event[1] as ENetPacketPeer).get_packet())
|
|
|
|
|
|
func test_a_client_joins() -> void:
|
|
var client: NetClient = _client(" Mara\tof the Reef ")
|
|
assert_eq(client.state, NetClient.CONNECTING)
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED), client.status_text())
|
|
assert_eq(client.handle, 1)
|
|
assert_eq(client.player_name, "Mara of the Reef")
|
|
assert_eq(client.match_seed, 4242)
|
|
assert_eq(client.max_players, Tuning.SOLO_PLAYERS)
|
|
assert_eq(client.tick_rate, Tuning.TICK_RATE_HZ)
|
|
assert_eq(client.server_version, NetProtocol.VERSION)
|
|
assert_eq(_events("joined"), [{"type": "joined", "handle": 1, "name": "Mara of the Reef"}] as Array[Dictionary])
|
|
assert_eq(server.player_count(), 1)
|
|
assert_eq(server.guest_count(), 0)
|
|
assert_eq(server.name_of(1), "Mara of the Reef")
|
|
assert_eq(client.status_text(), "Joined as Mara of the Reef.")
|
|
|
|
|
|
func test_a_version_mismatch_is_refused_clearly() -> void:
|
|
var client: NetClient = NetClient.new()
|
|
client.protocol_version = NetProtocol.VERSION + 1
|
|
clients.append(client)
|
|
assert_eq(client.connect_to(HOST, server.port(), "Mara"), OK)
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.CLOSED), client.state_name())
|
|
assert_eq(client.close_reason, NetProtocol.REASON_VERSION)
|
|
assert_eq(client.server_version, NetProtocol.VERSION)
|
|
var text: String = client.status_text()
|
|
assert_true(text.contains("older version") and text.contains("protocol %d" % NetProtocol.VERSION), text)
|
|
assert_eq(server.player_count(), 0)
|
|
assert_eq(_events("refused"), [{"type": "refused", "reason": NetProtocol.REASON_VERSION}] as Array[Dictionary])
|
|
assert_true(_events("joined").is_empty())
|
|
|
|
|
|
func test_a_full_server_says_so() -> void:
|
|
server.max_players = 2
|
|
var first: NetClient = _client("One")
|
|
var second: NetClient = _client("Two")
|
|
assert_true(_pump(func() -> bool: return first.state == NetClient.JOINED and second.state == NetClient.JOINED))
|
|
var third: NetClient = _client("Three")
|
|
assert_true(_pump(func() -> bool: return third.state == NetClient.CLOSED))
|
|
assert_eq(third.close_reason, NetProtocol.REASON_FULL)
|
|
assert_eq(third.status_text(), "The server is full.")
|
|
assert_eq(server.player_count(), 2)
|
|
# A place opens when someone leaves.
|
|
first.leave()
|
|
assert_true(_pump(func() -> bool: return server.player_count() == 1))
|
|
var fourth: NetClient = _client("Four")
|
|
assert_true(_pump(func() -> bool: return fourth.state == NetClient.JOINED))
|
|
assert_eq(fourth.handle, 3, "handles are never reused")
|
|
|
|
|
|
func test_duplicate_names_get_a_number() -> void:
|
|
var first: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return first.state == NetClient.JOINED))
|
|
var second: NetClient = _client("mara")
|
|
var third: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return second.state == NetClient.JOINED and third.state == NetClient.JOINED))
|
|
var names: Array[String] = [first.player_name, second.player_name, third.player_name]
|
|
names.sort()
|
|
assert_eq(names.size(), 3)
|
|
assert_true(names.has("Mara") and (names.has("mara 2") or names.has("Mara 2")), "names %s" % [names])
|
|
assert_ne(second.player_name.to_lower(), third.player_name.to_lower())
|
|
|
|
|
|
func test_a_clean_leave_is_seen_at_once() -> void:
|
|
server.peer_timeout_min_msec = 60000
|
|
server.peer_timeout_max_msec = 60000
|
|
var client: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED))
|
|
client.leave()
|
|
assert_eq(client.state, NetClient.CLOSED)
|
|
assert_eq(client.close_reason, NetProtocol.REASON_LEFT)
|
|
assert_true(_pump(func() -> bool: return not _events("left").is_empty(), 2000), "the server heard the goodbye, not a timeout")
|
|
assert_eq(_events("left"), [{"type": "left", "handle": 1, "name": "Mara", "reason": NetProtocol.REASON_LEFT}] as Array[Dictionary])
|
|
assert_eq(server.player_count(), 0)
|
|
|
|
|
|
func test_a_client_that_vanishes_times_out() -> void:
|
|
server.close()
|
|
server.peer_timeout_min_msec = 300
|
|
server.peer_timeout_max_msec = 600
|
|
assert_eq(server.listen(0, HOST), OK)
|
|
var client: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED))
|
|
# Pull the plug: no goodbye.
|
|
client._drop()
|
|
clients.clear()
|
|
assert_true(_pump(func() -> bool: return not _events("left").is_empty(), 8000), "the server noticed")
|
|
assert_eq(_events("left")[0]["reason"], NetProtocol.REASON_LOST)
|
|
assert_eq(server.player_count(), 0)
|
|
|
|
|
|
func test_a_silent_guest_is_dropped() -> void:
|
|
server.handshake_timeout_msec = 200
|
|
_raw()
|
|
var host: ENetConnection = raw_hosts[0]
|
|
assert_true(_pump(func() -> bool: return server.guest_count() == 1 or _raw_state[host]["disconnect_data"] >= 0))
|
|
assert_true(_pump(func() -> bool: return _raw_state[host]["disconnect_data"] >= 0), "the guest was disconnected")
|
|
assert_eq(_raw_state[host]["disconnect_data"], NetProtocol.REASON_HANDSHAKE_TIMEOUT)
|
|
assert_eq(NetProtocol.decode_reject(_raw_state[host]["packets"][0])["reason"], NetProtocol.REASON_HANDSHAKE_TIMEOUT)
|
|
assert_eq(server.guest_count(), 0)
|
|
assert_eq(server.player_count(), 0)
|
|
|
|
|
|
func test_malformed_handshakes_are_refused_and_the_server_carries_on() -> void:
|
|
var good: PackedByteArray = NetProtocol.encode_hello("Mara")
|
|
var huge: PackedByteArray = PackedByteArray()
|
|
huge.resize(60000)
|
|
huge.fill(0x41)
|
|
var wrong_magic: PackedByteArray = good.duplicate()
|
|
wrong_magic[1] = 0x00
|
|
var bad_name: PackedByteArray = good.slice(0, 7)
|
|
bad_name.append_array(PackedByteArray([2, 0xC3, 0x28]))
|
|
var packets: Array[PackedByteArray] = [
|
|
PackedByteArray([0]), PackedByteArray([NetProtocol.MSG_HELLO]), good.slice(0, 6), wrong_magic, huge, bad_name,
|
|
NetProtocol.encode_welcome(1, 1, 24, "Mara"), NetProtocol.encode_reject(NetProtocol.REASON_FULL),
|
|
]
|
|
for bytes: PackedByteArray in packets:
|
|
var peer: ENetPacketPeer = _raw()
|
|
peer.send(NetProtocol.CHANNEL_CONTROL, bytes, ENetPacketPeer.FLAG_RELIABLE)
|
|
# A good HELLO on the wrong channel is no handshake either.
|
|
var stray: ENetPacketPeer = _raw()
|
|
stray.send(NetProtocol.CHANNEL_STATE, good, ENetPacketPeer.FLAG_RELIABLE)
|
|
var expected: int = packets.size() + 1
|
|
assert_true(_pump(func() -> bool: return _events("refused").size() >= expected), "refused %d of %d" % [_events("refused").size(), expected])
|
|
for event: Dictionary in _events("refused"):
|
|
assert_eq(event["reason"], NetProtocol.REASON_BAD_HELLO)
|
|
assert_eq(server.bad_packets, expected)
|
|
assert_true(_pump(func() -> bool:
|
|
for host: ENetConnection in raw_hosts:
|
|
if _raw_state[host]["disconnect_data"] < 0:
|
|
return false
|
|
return true), "every bad peer was disconnected")
|
|
assert_eq(server.guest_count(), 0)
|
|
assert_eq(server.player_count(), 0)
|
|
# Still serving.
|
|
var client: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED))
|
|
|
|
|
|
func test_random_packets_from_guests_and_players_never_crash_the_server() -> void:
|
|
var rng: RandomNumberGenerator = RandomNumberGenerator.new()
|
|
rng.seed = 77
|
|
var member: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return member.state == NetClient.JOINED))
|
|
for _round: int in 12:
|
|
var peer: ENetPacketPeer = _raw()
|
|
for _packet: int in 6:
|
|
var bytes: PackedByteArray = PackedByteArray()
|
|
bytes.resize(rng.randi_range(0, 300))
|
|
for index: int in bytes.size():
|
|
bytes[index] = rng.randi_range(0, 255)
|
|
peer.send(rng.randi_range(0, 1), bytes, ENetPacketPeer.FLAG_RELIABLE)
|
|
# A player who keeps sending HELLOs and junk stays a player and changes nothing. The
|
|
# HELLOs are dropped; junk on the game channel goes to the owner, who can't read it.
|
|
for _packet: int in 20:
|
|
member._peer.send(NetProtocol.CHANNEL_CONTROL, NetProtocol.encode_hello("Someone Else"), ENetPacketPeer.FLAG_RELIABLE)
|
|
member._peer.send(NetProtocol.CHANNEL_STATE, PackedByteArray([1, 2, 3]), ENetPacketPeer.FLAG_RELIABLE)
|
|
assert_true(_pump(func() -> bool: return _events("refused").size() >= 12 and server.bad_packets >= 32 and handed >= 20), "refused %d, dropped %d, handed on %d" % [_events("refused").size(), server.bad_packets, handed])
|
|
assert_eq(handed, 20)
|
|
assert_eq(server.player_count(), 1)
|
|
assert_eq(server.name_of(1), "Mara")
|
|
assert_eq(_events("joined").size(), 1)
|
|
assert_eq(member.state, NetClient.JOINED)
|
|
|
|
|
|
func test_no_server_means_no_answer() -> void:
|
|
var port: int = server.port()
|
|
server.close()
|
|
var client: NetClient = NetClient.new()
|
|
client.connect_timeout_msec = 300
|
|
clients.append(client)
|
|
assert_eq(client.connect_to(HOST, port, "Mara"), OK)
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.CLOSED, 3000))
|
|
assert_eq(client.close_reason, NetProtocol.REASON_NO_ANSWER)
|
|
assert_true(client.status_text().contains("No answer"))
|
|
|
|
|
|
func test_a_server_shutting_down_tells_its_players() -> void:
|
|
var client: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED))
|
|
server.close()
|
|
assert_false(server.is_listening())
|
|
assert_eq(server.port(), 0)
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.CLOSED))
|
|
assert_eq(client.close_reason, NetProtocol.REASON_SHUTDOWN)
|
|
assert_eq(client.status_text(), "The server shut down.")
|
|
|
|
|
|
func test_a_kicked_player_is_told_and_removed() -> void:
|
|
var client: NetClient = _client("Mara")
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.JOINED))
|
|
assert_true(server.kick(1))
|
|
assert_eq(server.events[-1], {"type": "left", "handle": 1, "name": "Mara", "reason": NetProtocol.REASON_KICKED})
|
|
assert_false(server.kick(1))
|
|
assert_eq(server.player_count(), 0)
|
|
assert_true(_pump(func() -> bool: return client.state == NetClient.CLOSED))
|
|
assert_eq(client.close_reason, NetProtocol.REASON_KICKED)
|
|
assert_false(client.send(NetProtocol.CHANNEL_STATE, PackedByteArray([1]), false))
|
|
|
|
|
|
func test_two_servers_cannot_share_a_port() -> void:
|
|
var other: NetServer = NetServer.new()
|
|
expect_error("Couldn't create an ENet host")
|
|
assert_ne(other.listen(server.port(), HOST), OK)
|
|
assert_false(other.is_listening())
|
|
assert_true(server.is_listening())
|
|
|
|
|
|
func test_a_burst_of_connections_from_one_address_is_turned_away() -> void:
|
|
server.connect_limiter = SourceLimiter.new(3, 60000)
|
|
var burst: Array[NetClient] = []
|
|
for index: int in 5:
|
|
burst.append(_client("Wave %d" % index))
|
|
assert_true(_pump(func() -> bool:
|
|
var settled: int = 0
|
|
for client: NetClient in burst:
|
|
if client.state == NetClient.JOINED or client.state == NetClient.CLOSED:
|
|
settled += 1
|
|
return settled == burst.size()))
|
|
var joined: int = 0
|
|
for client: NetClient in burst:
|
|
if client.state == NetClient.JOINED:
|
|
joined += 1
|
|
else:
|
|
assert_eq(client.close_reason, NetProtocol.REASON_BUSY)
|
|
assert_eq(joined, 3)
|
|
assert_eq(server.player_count(), 3)
|
|
assert_true(NetProtocol.reason_text(NetProtocol.REASON_BUSY).begins_with("Too many connections"))
|
|
|
|
|
|
func test_handshakes_held_open_from_one_address_are_capped() -> void:
|
|
server.max_guests_per_address = 2
|
|
# Raw peers that connect and never say hello.
|
|
for _index: int in 4:
|
|
_raw()
|
|
assert_true(_pump(func() -> bool: return _events("refused").size() >= 2))
|
|
assert_eq(server.guest_count(), 2)
|
|
for event: Dictionary in _events("refused"):
|
|
assert_eq(event["reason"], NetProtocol.REASON_BUSY)
|
|
|
|
|
|
## N8: a server stopping with a grace sends its notice reliably, so clients behind a
|
|
## link that loses half its datagrams still hear it instead of timing out.
|
|
func test_a_graceful_shutdown_reaches_clients_on_a_lossy_link() -> void:
|
|
var relays: Array[UdpRelay] = []
|
|
var lossy: Array[NetClient] = []
|
|
for index: int in 4:
|
|
var relay: UdpRelay = UdpRelay.new(0.0, 0.0, 0.0, 11 + index)
|
|
assert_eq(relay.open(0, HOST, server.port()), OK)
|
|
relays.append(relay)
|
|
var client: NetClient = NetClient.new()
|
|
assert_eq(client.connect_to(HOST, relay.port(), "Lossy%d" % index), OK)
|
|
lossy.append(client)
|
|
var joined: Callable = func() -> bool:
|
|
for relay: UdpRelay in relays:
|
|
relay.poll()
|
|
for client: NetClient in lossy:
|
|
if client.state != NetClient.JOINED:
|
|
return false
|
|
return true
|
|
for _round: int in 2000:
|
|
server.poll()
|
|
for client: NetClient in lossy:
|
|
client.poll()
|
|
if joined.call():
|
|
break
|
|
OS.delay_msec(2)
|
|
assert_true(joined.call(), "every client joined")
|
|
for relay: UdpRelay in relays:
|
|
relay.loss = 0.5
|
|
# The clients go on being serviced in another thread while the server closes.
|
|
var stop: Array = [false]
|
|
var worker: Thread = Thread.new()
|
|
worker.start(func() -> void:
|
|
while not stop[0]:
|
|
for relay: UdpRelay in relays:
|
|
relay.poll()
|
|
for client: NetClient in lossy:
|
|
client.poll()
|
|
OS.delay_msec(2))
|
|
server.close(3000)
|
|
var started: int = Time.get_ticks_msec()
|
|
var all_closed: bool = false
|
|
while Time.get_ticks_msec() - started < 3000 and not all_closed:
|
|
OS.delay_msec(20)
|
|
all_closed = true
|
|
for client: NetClient in lossy:
|
|
all_closed = all_closed and client.state == NetClient.CLOSED
|
|
stop[0] = true
|
|
worker.wait_to_finish()
|
|
for client: NetClient in lossy:
|
|
assert_eq(client.state, NetClient.CLOSED)
|
|
assert_eq(client.close_reason, NetProtocol.REASON_SHUTDOWN, "%s was told the server stopped" % client.player_name)
|
|
for relay: UdpRelay in relays:
|
|
relay.close()
|