From 05c7568ec1fb45026d7138d95a40ebbf4c31d7fb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 12 Nov 2026 21:56:01 -0500 Subject: [PATCH] N8: 24-client network soak, and the server fixes it found make soak runs a real server and 24 headless clients in three processes for two whole matches in real time. Each client reaches the server through its own UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs unchanged over a bad link. SoakPlayer drives each client from what a client knows: it drops, glides, heads for high ground, harvests and tries to build. tools/soak_report.gd checks world hashes against the server's, errors, unreadable packets, every client playing every match with its connection intact, memory between matches, and the p99 tick with the network's share. make soak-selfcheck proves a drifted client world is caught. Server changes: --matches=N, --world-hash-every=N, tick timing that includes after_tick, and memory in match and progress lines. What the soak found and fixed: - Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the 12 ms budget, against a 2.8 ms simulation. Encoded records are now shared between clients for the tick, unchanged records skip the field diff, and clients are spread over the snapshot interval by handle. p99 is 7.3 ms. - ENet peers had a timeout limit of 0, which gives up on a peer as soon as any reliable message has waited the minimum timeout. Now ENet's default, 32. Not fixed: one or two clients still lose their connection about 15 s into the second match's drop in each run (see the issue), so the soak still fails and N8 stays open. Refs #39 Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 2 + Makefile | 12 +- docs/ARCHITECTURE.md | 29 +++- src/bots/soak_player.gd | 73 ++++++++ src/bots/soak_player.gd.uid | 1 + src/net/client_game.gd | 5 + src/net/net_client.gd | 2 +- src/net/net_protocol.gd | 8 + src/net/net_server.gd | 2 +- src/net/net_snapshot.gd | 48 +++--- src/net/snapshot_sender.gd | 29 +++- src/net/udp_relay.gd | 122 +++++++++++++ src/net/udp_relay.gd.uid | 1 + src/server/headless_server.gd | 31 +++- src/server/net_game.gd | 15 +- src/server/server_config.gd | 19 +- test/net/test_udp_relay.gd | 123 +++++++++++++ test/net/test_udp_relay.gd.uid | 1 + test/server/test_net_game.gd | 16 ++ test/server/test_server_config.gd | 11 ++ tools/soak.sh | 119 +++++++++++++ tools/soak_clients.gd | 276 ++++++++++++++++++++++++++++++ tools/soak_clients.gd.uid | 1 + tools/soak_report.gd | 227 ++++++++++++++++++++++++ tools/soak_report.gd.uid | 1 + 25 files changed, 1135 insertions(+), 39 deletions(-) create mode 100644 src/bots/soak_player.gd create mode 100644 src/bots/soak_player.gd.uid create mode 100644 src/net/udp_relay.gd create mode 100644 src/net/udp_relay.gd.uid create mode 100644 test/net/test_udp_relay.gd create mode 100644 test/net/test_udp_relay.gd.uid create mode 100755 tools/soak.sh create mode 100644 tools/soak_clients.gd create mode 100644 tools/soak_clients.gd.uid create mode 100644 tools/soak_report.gd create mode 100644 tools/soak_report.gd.uid diff --git a/CLAUDE.md b/CLAUDE.md index b3514e4..bfa9583 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -37,6 +37,8 @@ continuity. | `make discovery-check` | Starts a list server and a game server as processes and checks `--browse` finds the game both on the LAN (loopback) and through the list (about 15 s). | | `make fuzz` | Feeds 1,000,000 malformed messages (`FUZZ_MESSAGES=N`, `FUZZ_SEED=N`) to every network parser and fails on any engine error or memory growth (about 40 s). | | `make client-captures` | Renders what a network client sees on the barge, gliding and landed (`captures/client_.png`). Needs Docker. Look at them. | +| `make soak` | The 24-client network soak (N8): a real server and 24 headless `SoakPlayer` clients in three processes, each behind a `UdpRelay` adding 60 ms, +/-20 ms jitter and 2% loss each way, for two whole matches in real time (about 30 min, under `nice`). Fails on a world-hash mismatch, errors, unreadable packets, a client that didn't play every match, memory growth between matches, or a p99 tick over budget. `SOAK_MATCHES`, `SOAK_CLIENTS`, `SOAK_PROCS`, `SOAK_LATENCY`, `SOAK_JITTER`, `SOAK_LOSS` scale it. Results in `build/soak/report.txt`. | +| `make soak-selfcheck` | About a minute: tampers with one client's world and checks the soak's comparison catches it and only it. | | `make capture-test` | Proves the capture harness works and rejects a blank frame. Needs Docker, so it isn't part of `make check`. | Run one group of tests with `tools/run_tests.sh --root=res://test/sim` or diff --git a/Makefile b/Makefile index cbf0078..a6c3aae 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Tidewall build entry points. Everything runs headless with the repo-local engine. GODOT := tools/godot.sh -.PHONY: check import parse test test-selfcheck smoke capture-test terrain-capture binaries verify-engine bot-matches match match-captures client-check client-captures discovery-check fuzz +.PHONY: check import parse test test-selfcheck smoke capture-test terrain-capture binaries verify-engine bot-matches match match-captures client-check client-captures discovery-check fuzz soak soak-selfcheck ## Everything that must pass before a push. check: verify-engine binaries parse test test-selfcheck smoke @@ -72,3 +72,13 @@ capture-test: import ## Renders the terrain crack check and its control in the offscreen container. Needs Docker. terrain-capture: import @tools/terrain_capture_check.sh + +## The 24-client network soak: a real server and 24 headless clients behind a relay that +## adds latency, jitter and loss, for SOAK_MATCHES (default 2) whole matches in real +## time, about half an hour. Checks for desyncs, errors, leaks and the tick budget. +soak: import + @tools/soak.sh + +## About a minute: proves the soak catches a client whose world has drifted. +soak-selfcheck: import + @SOAK_SELFCHECK=1 tools/soak.sh diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 5914509..796d02d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -251,6 +251,12 @@ changed than fits, the client's own player goes first, then the nearest, and eac snapshot an update has waited counts as 6 m closer, so nothing waits for ever. What didn't fit is still owed and goes in a later snapshot. +**Cost.** Building 24 snapshots is the largest part of a busy server's tick, more +than the simulation. Records are made once a tick and shared, each (entity, mask) +encoding is made once a tick and shared, an unchanged record is skipped with one +native comparison, and clients are spread over the ticks of the snapshot interval by +handle (`NetGame.snapshot_due_for()`), so each tick builds half of them. + **Input.** The client answers on channel 1 with the snapshot tick it holds and its last few commands (`Tuning.NET_INPUT_REDUNDANCY`), oldest first. `NetGame` (`src/server/`) offers each tick's command to `ServerLoop.submit()` once, and echoes @@ -509,8 +515,29 @@ client is connecting, in the lobby or watching. | Budget | Target | |---|---| -| Server tick | Under **12 ms** with 24 players | +| Server tick | Under **12 ms** with 24 players, including the network's share (`make soak` checks the p99) | | Client | **60 fps** on a GTX 1060-class GPU at medium settings | | Terrain streaming | Under **8 ms** of main-thread work in any frame (measured headless; GPU upload unmeasured) | Software rendering in CI can't verify the client budget. It needs a real GPU playtest. + +## Soak testing + +`make soak` (`tools/soak.sh`) is the multiplayer gate: a real server process and 24 +headless clients in three processes, each client behind its own `UdpRelay`, a UDP +relay that delays, jitters and loses datagrams both ways, so ENet runs unchanged over +a bad link. `SoakPlayer` plays each client from what a client knows: it drops, glides, +heads for high ground, harvests and tries to build. + +Desync is checked by hash: the server, started with `--world-hash-every=N`, logs +`NetWorld.world_hash()` on the simulation ticks from each multiple of N to the snapshot +interval, and each client hashes its own world on the snapshots at those ticks when its +world messages have caught up exactly (`world_seq`) and it has no provisional pieces. +Tick 0 is never compared: it lasts the whole lobby while the world is set up. +`tools/soak_report.gd` compares them and checks errors, unreadable packets, that every +client played every match, the p99 tick, and memory from one match to the next. +`make soak-selfcheck` tampers with one client's world to prove a drift is caught. + +A client process holds back a new match's notice until it has made that client's +island, at most one a frame: a dozen clients making islands in one frame starved +their connections of service for long enough that ENet dropped them. diff --git a/src/bots/soak_player.gd b/src/bots/soak_player.gd new file mode 100644 index 0000000..bea6c0f --- /dev/null +++ b/src/bots/soak_player.gd @@ -0,0 +1,73 @@ +## The script a soak-test client plays by. It only sees what a network client sees: its +## own predicted player and its copy of the island. It leaps from the barge over land, +## glides, heads for the island's highest ground (where the tide leaves people last), +## swings the boathook at whatever is in front of it, which harvests trees and crates, +## and every few seconds tries to put up a wall. So a soak match has movement, world +## changes and building from every client, not just walking. +## +## Each client's script is offset by its `number`, so they don't all act on the same tick. +class_name SoakPlayer +extends RefCounted + +const TICKS: int = Tuning.TICK_RATE_HZ +## Within this many metres of the high ground, wander instead of heading for it. +const NEAR_TOP_METRES: float = 24.0 +const BUILD: int = PlayerCommand.BUTTON_BUILD | PlayerCommand.BUTTON_FIRE + +var number: int = 0 +## Build attempts made, for the report. +var build_attempts: int = 0 + +var _world: Simulation = null +var _top: Vector2 = Vector2.ZERO + + +func _init(client_number: int) -> void: + number = client_number + + +## The command for `tick`, for the player `me` on the client's `world`. +func command(tick: int, me: SimEntity, world: Simulation) -> PlayerCommand: + if world != _world: + _world = world + _top = _highest_point(world.ground) + var seconds: float = float(tick + number * 37) / TICKS + var to_top: Vector2 = _top - Vector2(me.position.x, me.position.z) + var yaw: float = atan2(to_top.x, to_top.y) + if me.drop_state == Drop.ABOARD: + var wait: int = (3 + number % 8) * TICKS + var over_land: bool = world.ground.terrain_height(me.position.x, me.position.z) > Tuning.ISLAND_BEACH_TOP + 2.0 + return PlayerCommand.from_input(tick, Vector2.ZERO, yaw, 0.0, PlayerCommand.BUTTON_JUMP if tick > wait and over_land else 0) + if me.drop_state != Drop.NONE: + return PlayerCommand.from_input(tick, Vector2(0.0, 1.0), yaw, 0.0) + var phase: int = (tick + number * 11) % (6 * TICKS) + if phase == 0 and Building.is_allowed(me): + build_attempts += 1 + # A wall on one side or another of the cell it stands in. + return PlayerCommand.from_input(tick, Vector2.ZERO, (number + build_attempts) % 4 * PI * 0.5, 0.0, BUILD, StructureSet.WALL, Materials.DRIFTWOOD) + var move: Vector2 = Vector2(0.0, 1.0) + if to_top.length() < NEAR_TOP_METRES: + yaw += sin(seconds * 0.5) * 1.5 + move = Vector2(0.0, 1.0) if fmod(seconds, 8.0) < 5.0 else Vector2.ZERO + var buttons: int = 0 + if phase > 3 * TICKS: + # Swing at what is in front: trees and crates give materials to build with. + buttons |= PlayerCommand.BUTTON_FIRE + move = Vector2.ZERO + if (tick + number) % (3 * TICKS) == 0: + buttons |= PlayerCommand.BUTTON_JUMP + return PlayerCommand.from_input(tick, move, yaw, deg_to_rad(-10.0), buttons, 0) + + +static func _highest_point(ground: SimGround) -> Vector2: + var best: Vector2 = Vector2.ONE * Tuning.WORLD_SIZE_METRES * 0.5 + var best_height: float = -INF + var step: float = SafeGround.STRIDE_METRES + var cells: int = floori(Tuning.WORLD_SIZE_METRES / step) + for gz: int in cells: + for gx: int in cells: + var height: float = ground.terrain_height(gx * step, gz * step) + if height > best_height: + best_height = height + best = Vector2(gx * step, gz * step) + return best diff --git a/src/bots/soak_player.gd.uid b/src/bots/soak_player.gd.uid new file mode 100644 index 0000000..826cb7f --- /dev/null +++ b/src/bots/soak_player.gd.uid @@ -0,0 +1 @@ +uid://crrcpgj0w1l6j diff --git a/src/net/client_game.gd b/src/net/client_game.gd index 8ebaa40..da7bcac 100644 --- a/src/net/client_game.gd +++ b/src/net/client_game.gd @@ -44,6 +44,9 @@ var outbox: Array[PackedByteArray] = [] var net: NetClient = null ## Packets that made no sense, so far. var bad_packets: int = 0 +## Called after each snapshot has been taken in, with no arguments. Soak tests use it +## to compare the client's world with the server's at that moment. +var on_snapshot: Callable = Callable() var _recent: Array[PlayerCommand] = [] var _ack_due: bool = false @@ -91,6 +94,8 @@ func receive(channel: int, bytes: PackedByteArray) -> void: _take_notice(notice) elif receiver.apply(bytes): _take_snapshot() + if on_snapshot.is_valid(): + on_snapshot.call() ## Runs the client for `delta` seconds: makes and predicts the commands that are due, diff --git a/src/net/net_client.gd b/src/net/net_client.gd index 26746de..191d945 100644 --- a/src/net/net_client.gd +++ b/src/net/net_client.gd @@ -82,7 +82,7 @@ func poll(now_msec: int = -1) -> void: break match kind: ENetConnection.EVENT_CONNECT: - _peer.set_timeout(0, peer_timeout_min_msec, peer_timeout_max_msec) + _peer.set_timeout(NetProtocol.PEER_TIMEOUT_LIMIT, peer_timeout_min_msec, peer_timeout_max_msec) _peer.send(NetProtocol.CHANNEL_CONTROL, NetProtocol.encode_hello(_wanted_name, protocol_version), ENetPacketPeer.FLAG_RELIABLE) state = HANDSHAKING _since_msec = now diff --git a/src/net/net_protocol.gd b/src/net/net_protocol.gd index 4278cb8..6d2ef4b 100644 --- a/src/net/net_protocol.gd +++ b/src/net/net_protocol.gd @@ -28,6 +28,14 @@ const MSG_WELCOME: int = 2 const MSG_REJECT: int = 3 ## Why a connection ended. Sent in REJECT and as disconnect data. +## ENet's timeout limit for every peer: a reliable message is given up on after this +## many round-trip timeouts' worth of resends (and at least the peer's minimum timeout), +## or after its maximum timeout. ENet's own default. 0, which this once was, gives up as +## soon as any reliable message has waited the minimum timeout: over a link losing 2% +## of datagrams each way, a run of unlucky resends then dropped a few soak clients a +## match (issue N8). +const PEER_TIMEOUT_LIMIT: int = 32 + const REASON_NONE: int = 0 const REASON_VERSION: int = 1 const REASON_FULL: int = 2 diff --git a/src/net/net_server.gd b/src/net/net_server.gd index 1a38449..1893f5a 100644 --- a/src/net/net_server.gd +++ b/src/net/net_server.gd @@ -107,7 +107,7 @@ func poll(now_msec: int = -1) -> void: var peer: ENetPacketPeer = event[1] match kind: ENetConnection.EVENT_CONNECT: - peer.set_timeout(0, peer_timeout_min_msec, peer_timeout_max_msec) + peer.set_timeout(NetProtocol.PEER_TIMEOUT_LIMIT, peer_timeout_min_msec, peer_timeout_max_msec) var address: String = peer.get_remote_address() if not connect_limiter.allow(address, now) or _guests_from(address) >= max_guests_per_address: _refuse(peer, NetProtocol.REASON_BUSY) diff --git a/src/net/net_snapshot.gd b/src/net/net_snapshot.gd index 5f3e9b2..58d22b9 100644 --- a/src/net/net_snapshot.gd +++ b/src/net/net_snapshot.gd @@ -263,26 +263,34 @@ static func encode_snapshot(header: Dictionary, removed: PackedInt32Array, recor _put_varint(out, entity_id) out.put_u16(records.size()) for entry: Array in records: - var record: PackedInt32Array = entry[2] - var archetype: int = record[R_ARCHETYPE] - var mask: int = entry[1] - var types: Array = _TYPES[archetype] - var groups: Array = _GROUPS[archetype] - _put_varint(out, int(entry[0])) - out.put_u8(archetype << 5 | mask) - for group: int in groups.size(): - if mask & (1 << group) == 0: - continue - for index: int in groups[group]: - match types[index]: - _U8: - out.put_u8(record[index]) - _S8: - out.put_8(record[index]) - _U16: - out.put_u16(record[index]) - _U32: - out.put_u32(record[index]) + # A record may come already encoded (encode_record()), shared between clients. + out.put_data(entry[3] if entry.size() > 3 else encode_record(int(entry[0]), int(entry[1]), entry[2])) + return out.data_array + + +## One record as a snapshot carries it: the entity ID, the archetype and mask byte, and +## the fields of each group in `mask`. The same for every client that is sent it. +static func encode_record(entity_id: int, mask: int, record: PackedInt32Array) -> PackedByteArray: + var out: StreamPeerBuffer = StreamPeerBuffer.new() + out.big_endian = false + var archetype: int = record[R_ARCHETYPE] + var types: Array = _TYPES[archetype] + var groups: Array = _GROUPS[archetype] + _put_varint(out, entity_id) + out.put_u8(archetype << 5 | mask) + for group: int in groups.size(): + if mask & (1 << group) == 0: + continue + for index: int in groups[group]: + match types[index]: + _U8: + out.put_u8(record[index]) + _S8: + out.put_8(record[index]) + _U16: + out.put_u16(record[index]) + _U32: + out.put_u32(record[index]) return out.data_array diff --git a/src/net/snapshot_sender.gd b/src/net/snapshot_sender.gd index 0d550fa..c2ad652 100644 --- a/src/net/snapshot_sender.gd +++ b/src/net/snapshot_sender.gd @@ -24,6 +24,10 @@ var _records: Dictionary = {} var _records_tick: int = -1 ## Every entity at that tick, listed once for all clients. var _entities: Array[SimEntity] = [] +## Also shared for the tick: entity ID * 64 + mask -> the record encoded with that mask +## (NetSnapshot.encode_record()), and entity ID -> its record's position. +var _encoded: Dictionary = {} +var _positions: Dictionary = {} func add_client(handle: int) -> void: @@ -68,6 +72,8 @@ func reset() -> void: client["last_sent"] = {} client["command_tick"] = -1 _records.clear() + _encoded.clear() + _positions.clear() _entities.clear() _records_tick = -1 @@ -123,6 +129,8 @@ func build(handle: int, sim: Simulation, viewer_id: int, tick: int, match_phase: var viewer: SimEntity = sim.entities.find(viewer_id) if viewer_id > 0 else null if _records_tick != tick: _records.clear() + _encoded.clear() + _positions.clear() _records_tick = tick _entities = sim.entities.entities() @@ -146,13 +154,19 @@ func build(handle: int, sim: Simulation, viewer_id: int, tick: int, match_phase: visible[entity_id] = true var mask: int = NetSnapshot.full_mask(record[NetSnapshot.R_ARCHETYPE]) if base.has(entity_id): - mask = NetSnapshot.diff_mask(base[entity_id], record) + var before: PackedInt32Array = base[entity_id] + # Most entities, standing still, compare equal: skip the field-by-field diff. + if before == record: + continue + mask = NetSnapshot.diff_mask(before, record) if mask == 0: continue var score: float = -1.0e9 if entity_id != viewer.id: var waited: float = float(tick - int(last_sent.get(entity_id, tick - interval * 20))) / interval - score = NetSnapshot.position_of(record).distance_to(viewer.position) - waited * Tuning.NET_STALE_METRES_PER_SNAPSHOT + if not _positions.has(entity_id): + _positions[entity_id] = NetSnapshot.position_of(record) + score = (_positions[entity_id] as Vector3).distance_to(viewer.position) - waited * Tuning.NET_STALE_METRES_PER_SNAPSHOT changed.append([score, entity_id, mask, record]) changed.sort_custom(func(a: Array, b: Array) -> bool: return a[0] < b[0] if a[0] != b[0] else a[1] < b[1]) @@ -172,12 +186,15 @@ func build(handle: int, sim: Simulation, viewer_id: int, tick: int, match_phase: var records: Array = [] for entry: Array in changed: var record: PackedInt32Array = entry[3] - var cost: int = NetSnapshot.record_bytes(entry[1], record[NetSnapshot.R_ARCHETYPE], entry[2]) - if cost > budget: + var key: int = int(entry[1]) * 64 + int(entry[2]) + if not _encoded.has(key): + _encoded[key] = NetSnapshot.encode_record(entry[1], entry[2], record) + var encoded: PackedByteArray = _encoded[key] + if encoded.size() > budget: client["deferred"] += 1 continue - budget -= cost - records.append([entry[1], entry[2], record]) + budget -= encoded.size() + records.append([entry[1], entry[2], record, encoded]) state[entry[1]] = record last_sent[entry[1]] = tick diff --git a/src/net/udp_relay.gd b/src/net/udp_relay.gd new file mode 100644 index 0000000..fac8352 --- /dev/null +++ b/src/net/udp_relay.gd @@ -0,0 +1,122 @@ +## A bad network in a box, for soak tests: a UDP relay that sits between one client and +## a server and passes datagrams both ways after a delay of `latency_ms` plus or minus +## `jitter_ms`, losing a `loss` fraction of them. Packets can overtake each other, as on +## a real link. Seeded, so a run can be repeated. +## +## The client connects to the relay's port instead of the server's. The first datagram +## to arrive there says where the client is; later ones from anywhere else are ignored. +## ENet runs over UDP, so the game's own transport is tested unchanged. +## +## Call poll() often (every frame). Delays are only as fine as the gap between polls. +class_name UdpRelay +extends RefCounted + +var latency_ms: float = 0.0 +var jitter_ms: float = 0.0 +var loss: float = 0.0 + +## Datagrams and payload bytes, each way, that came in, and the datagrams lost. +## "up" is client to server, "down" server to client. +var up_packets: int = 0 +var up_bytes: int = 0 +var up_lost: int = 0 +var down_packets: int = 0 +var down_bytes: int = 0 +var down_lost: int = 0 +## When a datagram last came in from each side, in msec (Time.get_ticks_msec()), or -1. +var last_up_msec: int = -1 +var last_down_msec: int = -1 + +var _rng: RandomNumberGenerator = RandomNumberGenerator.new() +var _downstream: PacketPeerUDP = PacketPeerUDP.new() +var _upstream: PacketPeerUDP = PacketPeerUDP.new() +var _client_ip: String = "" +var _client_port: int = 0 +## [due msec, true when bound for the server, bytes], in no particular order. +var _queue: Array = [] + + +func _init(latency: float = 0.0, jitter: float = 0.0, loss_fraction: float = 0.0, relay_seed: int = 1) -> void: + latency_ms = latency + jitter_ms = jitter + loss = loss_fraction + _rng.seed = relay_seed + + +## Listens for the client on `port` (0 picks a free one) and forwards to the server at +## `server_ip`:`server_port`. +func open(port: int, server_ip: String, server_port: int, bind_address: String = "127.0.0.1") -> Error: + var error: Error = _downstream.bind(port, bind_address) + if error != OK: + return error + error = _upstream.connect_to_host(server_ip, server_port) + if error != OK: + _downstream.close() + return error + + +## The port the client should connect to. +func port() -> int: + return _downstream.get_local_port() + + +func close() -> void: + _downstream.close() + _upstream.close() + _queue.clear() + + +## Takes what has arrived on both sides and sends on whatever is due. +func poll(now_msec: int = -1) -> void: + var now: int = now_msec if now_msec >= 0 else Time.get_ticks_msec() + while _downstream.get_available_packet_count() > 0: + var bytes: PackedByteArray = _downstream.get_packet() + var ip: String = _downstream.get_packet_ip() + var from_port: int = _downstream.get_packet_port() + if _client_ip.is_empty(): + _client_ip = ip + _client_port = from_port + _downstream.set_dest_address(_client_ip, _client_port) + elif ip != _client_ip or from_port != _client_port: + continue + up_packets += 1 + up_bytes += bytes.size() + last_up_msec = now + if _rng.randf() < loss: + up_lost += 1 + else: + _queue.append([now + _delay(), true, bytes]) + while _upstream.get_available_packet_count() > 0: + var bytes: PackedByteArray = _upstream.get_packet() + down_packets += 1 + down_bytes += bytes.size() + last_down_msec = now + if _rng.randf() < loss: + down_lost += 1 + else: + _queue.append([now + _delay(), false, bytes]) + if _queue.is_empty(): + return + var waiting: Array = [] + var due: Array = [] + for entry: Array in _queue: + if int(entry[0]) <= now: + due.append(entry) + else: + waiting.append(entry) + _queue = waiting + due.sort_custom(func(a: Array, b: Array) -> bool: return int(a[0]) < int(b[0])) + for entry: Array in due: + if entry[1]: + _upstream.put_packet(entry[2]) + elif not _client_ip.is_empty(): + _downstream.put_packet(entry[2]) + + +## Datagrams waiting to be sent on. +func in_flight() -> int: + return _queue.size() + + +func _delay() -> int: + return maxi(roundi(latency_ms + _rng.randf_range(-jitter_ms, jitter_ms)), 0) diff --git a/src/net/udp_relay.gd.uid b/src/net/udp_relay.gd.uid new file mode 100644 index 0000000..cfa545b --- /dev/null +++ b/src/net/udp_relay.gd.uid @@ -0,0 +1 @@ +uid://cq3gv5durd8x6 diff --git a/src/server/headless_server.gd b/src/server/headless_server.gd index c11c12c..e05ae3d 100644 --- a/src/server/headless_server.gd +++ b/src/server/headless_server.gd @@ -26,6 +26,10 @@ var _errors: ErrorCounter = ErrorCounter.new() var _accumulator: float = 0.0 var _started_usec: int = 0 var _finished: bool = false +var _matches_ended: int = 0 +## When listening: microseconds each tick took including the network's share (after_tick: +## world messages and snapshots), which ServerLoop's own timing leaves out. +var _net_tick_usec: PackedInt64Array = PackedInt64Array() func _init(server_config: ServerConfig) -> void: @@ -91,10 +95,10 @@ func _process(delta: float) -> void: # Too far behind to catch up: drop the backlog rather than spiral. _accumulator = minf(_accumulator, 1.0 / Tuning.TICK_RATE_HZ) for _index: int in due: - if config.ticks > 0 and loop.ticks >= config.ticks: + if _done(): break _tick() - if config.ticks > 0 and loop.ticks >= config.ticks: + if _done(): _finish(0) @@ -104,6 +108,11 @@ func _exit_tree() -> void: _finish(0) +## True once the run has done what it was asked: its ticks, or its matches. +func _done() -> bool: + return (config.ticks > 0 and loop.ticks >= config.ticks) or (config.matches > 0 and _matches_ended >= config.matches) + + func _open_beacon() -> void: if not config.lan and config.lists.is_empty(): return @@ -131,19 +140,26 @@ func _open_beacon() -> void: func _tick() -> void: if config.fault == "script-error" and loop.ticks == FAULT_TICK: _deliberate_script_error() + var began: int = Time.get_ticks_usec() loop.run_tick() if game != null: game.after_tick() + _net_tick_usec.append(Time.get_ticks_usec() - began) + if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS: + _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim)}) for event: Dictionary in loop.match_flow.events: if event["type"] in ["phase", "match_started", "match_ended"]: - _log("match", event) + var fields: Dictionary = event.duplicate() + fields["mem_bytes"] = OS.get_static_memory_usage() + _log("match", fields) elif event["type"] == "placed": _log("eliminated", event) if event["type"] == "match_ended": _log("results", {"match": event["match"], "summary": loop.match_flow.summary()}) + _matches_ended += 1 if config.log_every > 0 and loop.ticks % config.log_every == 0: var stats: Dictionary = loop.timing_stats() - _log("progress", {"tick": loop.ticks, "p95_us": stats["p95_us"], "max_us": stats["max_us"]}) + _log("progress", {"tick": loop.ticks, "p95_us": stats["p95_us"], "max_us": stats["max_us"], "mem_bytes": OS.get_static_memory_usage()}) func _finish(requested_code: int) -> void: @@ -157,6 +173,13 @@ func _finish(requested_code: int) -> void: var stats: Dictionary = loop.timing_stats() stats["state_hash"] = loop.sim.state_hash() stats["entities"] = loop.sim.entities.count() + if not _net_tick_usec.is_empty(): + var sorted: PackedInt64Array = _net_tick_usec.duplicate() + sorted.sort() + stats["with_net_p50_us"] = ServerLoop._percentile(sorted, 0.50) + stats["with_net_p99_us"] = ServerLoop._percentile(sorted, 0.99) + stats["with_net_max_us"] = sorted[sorted.size() - 1] + stats["mem_bytes"] = OS.get_static_memory_usage() _log("stats", stats) if code == 0 and _errors.count() > 0: code = 1 diff --git a/src/server/net_game.gd b/src/server/net_game.gd index 337e66e..b6d9393 100644 --- a/src/server/net_game.gd +++ b/src/server/net_game.gd @@ -138,18 +138,25 @@ func after_tick() -> void: if not bytes.is_empty(): own_changes[handle] = bytes net.send(handle, NetProtocol.CHANNEL_CONTROL, bytes, true) - if not snapshot_due(): - return for handle: int in net.handles(): - if sender.has_client(handle): + if sender.has_client(handle) and snapshot_due_for(handle): net.send(handle, NetProtocol.CHANNEL_STATE, snapshot_for(handle), false) -## True on the ticks a snapshot goes out. +## True on the ticks a snapshot goes out to an owner that carries packets for a single +## client itself (LocalServer and tests). func snapshot_due() -> bool: return loop.ticks % Tuning.NET_SNAPSHOT_INTERVAL_TICKS == 0 +## True on the ticks a networked client's snapshot goes out. Clients are spread over +## the ticks of the interval by handle, so the cost of building snapshots, the largest +## share of a busy server's tick, is split between them instead of landing on one. +func snapshot_due_for(handle: int) -> bool: + var interval: int = Tuning.NET_SNAPSHOT_INTERVAL_TICKS + return (loop.ticks + posmod(handle, interval)) % interval == 0 + + ## The snapshot for a participant as things stand: what they may see, and their own ## block while they are playing. func snapshot_for(handle: int) -> PackedByteArray: diff --git a/src/server/server_config.gd b/src/server/server_config.gd index 38e793a..4cfb9e7 100644 --- a/src/server/server_config.gd +++ b/src/server/server_config.gd @@ -2,7 +2,7 @@ ## ## tools/godot.sh -- --server [--listen] [--port=N] [--seed=N] [--players=N] [--bots=N] ## [--bot-difficulty=mixed|easy|normal|hard] [--ticks=N] -## [--unthrottled] [--log-every=N] +## [--unthrottled] [--log-every=N] [--matches=N] [--world-hash-every=N] ## [--name=NAME] [--list=HOST[:PORT]]... [--no-lan] ## ## With `--listen` the server takes network players on UDP `--port`, waits in the lobby @@ -38,6 +38,12 @@ var bots: int = 0 var bot_difficulty: int = -1 ## Ticks to run before exiting. 0 runs until the process is stopped. var ticks: int = 0 +## Stops after this many matches have ended. 0 doesn't stop for matches. +var matches: int = 0 +## On the simulation ticks from each multiple of this many up to the snapshot interval +## (never tick 0, which lasts the whole lobby), logs NetWorld.world_hash() of the match's +## world, so a soak test can compare it with its clients' copies. 0 turns it off. +var world_hash_every: int = 0 ## Runs ticks back to back instead of at the real tick rate. For tests and smoke runs. var unthrottled: bool = false ## Logs a progress line every this many ticks. 0 turns progress lines off. @@ -81,6 +87,7 @@ func to_dictionary() -> Dictionary: "bots": bots, "bot_difficulty": "mixed" if bot_difficulty < 0 else String(Tuning.BOT_DIFFICULTIES[bot_difficulty]["name"]), "ticks": ticks, + "matches": matches, "unthrottled": unthrottled, "name": server_name, "lan": lan and listen, @@ -124,6 +131,12 @@ func _parse_one(arg: String) -> void: "--log-every": if _check_int(key, value): log_every = value.to_int() + "--matches": + if _check_int(key, value): + matches = value.to_int() + "--world-hash-every": + if _check_int(key, value): + world_hash_every = value.to_int() "--bot-difficulty": var found: bool = value == "mixed" for index: int in Tuning.BOT_DIFFICULTIES.size(): @@ -195,6 +208,10 @@ func _validate() -> void: errors.append("--ticks can't be negative") if log_every < 0: errors.append("--log-every can't be negative") + if matches < 0 or world_hash_every < 0: + errors.append("--matches and --world-hash-every can't be negative") + if world_hash_every > 0 and not listen: + errors.append("--world-hash-every is for a server started with --listen") if not listen and (not lists.is_empty() or not lan): errors.append("--list and --no-lan are for a server started with --listen") if fault not in FAULTS: diff --git a/test/net/test_udp_relay.gd b/test/net/test_udp_relay.gd new file mode 100644 index 0000000..6917308 --- /dev/null +++ b/test/net/test_udp_relay.gd @@ -0,0 +1,123 @@ +extends TestCase + +## Issue N8: the soak test's bad network. UdpRelay delays, loses and counts datagrams +## both ways, ignores strangers, and the game's real transport still joins through it. + +const HOST: String = "127.0.0.1" + +var relay: UdpRelay +var server_socket: PacketPeerUDP +var client_socket: PacketPeerUDP + + +func before_each() -> void: + server_socket = PacketPeerUDP.new() + assert_eq(server_socket.bind(0, HOST), OK) + client_socket = PacketPeerUDP.new() + relay = null + + +func after_each() -> void: + if relay != null: + relay.close() + server_socket.close() + client_socket.close() + + +func _open(latency: float, jitter: float, loss: float) -> void: + relay = UdpRelay.new(latency, jitter, loss, 7) + assert_eq(relay.open(0, HOST, server_socket.get_local_port()), OK) + assert_eq(client_socket.connect_to_host(HOST, relay.port()), OK) + + +## Waits a little for loopback datagrams to land, then polls the relay at `now`. +func _settle_and_poll(now: int) -> void: + OS.delay_msec(20) + relay.poll(now) + + +func test_delays_both_ways() -> void: + _open(100.0, 0.0, 0.0) + client_socket.put_packet(PackedByteArray([1, 2, 3])) + _settle_and_poll(1000) + assert_eq(relay.in_flight(), 1, "held, not passed on at once") + relay.poll(1099) + OS.delay_msec(20) + assert_eq(server_socket.get_available_packet_count(), 0, "not before its time") + relay.poll(1100) + OS.delay_msec(20) + assert_eq(server_socket.get_available_packet_count(), 1) + assert_eq(server_socket.get_packet(), PackedByteArray([1, 2, 3])) + # The server answers whoever sent it: the relay's other side. + server_socket.set_dest_address(server_socket.get_packet_ip(), server_socket.get_packet_port()) + server_socket.put_packet(PackedByteArray([9])) + _settle_and_poll(2000) + relay.poll(2100) + OS.delay_msec(20) + assert_eq(client_socket.get_available_packet_count(), 1) + assert_eq(client_socket.get_packet(), PackedByteArray([9])) + assert_eq([relay.up_packets, relay.up_bytes, relay.down_packets, relay.down_bytes], [1, 3, 1, 1]) + + +func test_loss_drops_and_counts() -> void: + _open(0.0, 0.0, 1.0) + for index: int in 5: + client_socket.put_packet(PackedByteArray([index])) + _settle_and_poll(0) + relay.poll(1) + OS.delay_msec(20) + assert_eq(relay.up_lost, 5) + assert_eq(relay.in_flight(), 0) + assert_eq(server_socket.get_available_packet_count(), 0) + + +func test_jitter_stays_in_range() -> void: + _open(50.0, 20.0, 0.0) + for index: int in 40: + client_socket.put_packet(PackedByteArray([index])) + _settle_and_poll(0) + assert_eq(relay.in_flight(), 40) + relay.poll(29) + OS.delay_msec(20) + assert_eq(server_socket.get_available_packet_count(), 0, "none sooner than latency less jitter") + relay.poll(70) + OS.delay_msec(20) + assert_eq(server_socket.get_available_packet_count(), 40, "all by latency plus jitter") + + +func test_ignores_a_second_sender() -> void: + _open(0.0, 0.0, 0.0) + client_socket.put_packet(PackedByteArray([1])) + _settle_and_poll(0) + var stranger: PacketPeerUDP = PacketPeerUDP.new() + assert_eq(stranger.connect_to_host(HOST, relay.port()), OK) + stranger.put_packet(PackedByteArray([2])) + _settle_and_poll(1) + relay.poll(2) + OS.delay_msec(20) + stranger.close() + assert_eq(relay.up_packets, 1) + assert_eq(server_socket.get_available_packet_count(), 1) + assert_eq(server_socket.get_packet(), PackedByteArray([1])) + + +func test_real_transport_joins_through_a_bad_link() -> void: + server_socket.close() + var server: NetServer = NetServer.new() + server.match_seed = 99 + assert_eq(server.listen(0, HOST), OK) + relay = UdpRelay.new(40.0, 15.0, 0.1, 3) + assert_eq(relay.open(0, HOST, server.port()), OK) + var client: NetClient = NetClient.new() + assert_eq(client.connect_to(HOST, relay.port(), "Wren"), OK) + var started: int = Time.get_ticks_msec() + while client.state != NetClient.JOINED and Time.get_ticks_msec() - started < 8000: + server.poll() + relay.poll() + client.poll() + OS.delay_msec(2) + assert_eq(client.state_name(), "joined") + assert_eq(client.match_seed, 99) + assert_true(relay.up_packets > 0 and relay.down_packets > 0) + client.leave() + server.close() diff --git a/test/net/test_udp_relay.gd.uid b/test/net/test_udp_relay.gd.uid new file mode 100644 index 0000000..5e92d41 --- /dev/null +++ b/test/net/test_udp_relay.gd.uid @@ -0,0 +1 @@ +uid://bue7ocmcipqfq diff --git a/test/server/test_net_game.gd b/test/server/test_net_game.gd index fa72367..4497ede 100644 --- a/test/server/test_net_game.gd +++ b/test/server/test_net_game.gd @@ -212,3 +212,19 @@ func test_junk_on_the_game_channel_is_counted_and_changes_nothing() -> void: assert_true(_pump(func() -> bool: return game.bad_inputs >= 20), "%d counted" % game.bad_inputs) assert_eq(mara.state, NetClient.JOINED) assert_true(loop.match_flow.in_match()) + + +func test_snapshots_are_spread_over_the_interval() -> void: + # Issue N8: building every client's snapshot on one tick put the whole cost there. + var interval: int = Tuning.NET_SNAPSHOT_INTERVAL_TICKS + var hits: Dictionary = {} + for _tick: int in interval * 10: + loop.run_tick() + var due: int = 0 + for handle: int in range(1, 25): + if game.snapshot_due_for(handle): + due += 1 + hits[handle] = int(hits.get(handle, 0)) + 1 + assert_eq(due, 24 / interval, "an even share of 24 clients on every tick") + for handle: int in range(1, 25): + assert_eq(hits.get(handle, 0), 10, "one snapshot every interval for handle %d" % handle) diff --git a/test/server/test_server_config.gd b/test/server/test_server_config.gd index 4cd7c31..3cae9ba 100644 --- a/test/server/test_server_config.gd +++ b/test/server/test_server_config.gd @@ -108,3 +108,14 @@ func test_bad_discovery_options() -> void: assert_false(_parse(["--server", "--listen", wrong]).is_valid(), wrong) assert_false(_parse(["--server", "--list=192.0.2.4"]).is_valid(), "a list server needs --listen") assert_false(_parse(["--server", "--no-lan"]).is_valid()) + + +func test_soak_options() -> void: + var config: ServerConfig = _parse(["--server", "--listen", "--matches=2", "--world-hash-every=300"]) + assert_true(config.is_valid(), str(config.errors)) + assert_eq(config.matches, 2) + assert_eq(config.world_hash_every, 300) + assert_eq(config.to_dictionary()["matches"], 2) + assert_false(_parse(["--server", "--world-hash-every=300"]).is_valid(), "hashes are only logged when listening") + assert_false(_parse(["--server", "--matches=-1"]).is_valid()) + diff --git a/tools/soak.sh b/tools/soak.sh new file mode 100755 index 0000000..738b562 --- /dev/null +++ b/tools/soak.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +# The 24-client network soak (issue N8). Starts a real dedicated server process and +# SOAK_PROCS client processes, which between them run SOAK_CLIENTS headless clients. +# Every client reaches the server through its own UdpRelay, which adds SOAK_LATENCY ms +# of delay, +/- SOAK_JITTER ms of jitter and SOAK_LOSS of loss each way. The clients +# are played by SoakPlayer: they drop, glide, run for high ground, harvest and build. +# The server plays SOAK_MATCHES whole matches in real time and stops, which ends the +# clients, and tools/soak_report.gd decides whether it all passed: see that file. +# +# A full match is about 15 minutes of real time, so the default run (two matches) takes +# about half an hour. Everything runs under nice. If the host's available memory falls +# under 2 GB the run is stopped and fails, rather than squeezing other work. +# +# tools/soak.sh SOAK_MATCHES=1 SOAK_CLIENTS=8 ... to scale it +# SOAK_SELFCHECK=1 tools/soak.sh a minute's run with one client's world tampered +# with, which must be caught (make soak-selfcheck) +set -uo pipefail + +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" +selfcheck="${SOAK_SELFCHECK:-0}" +clients="${SOAK_CLIENTS:-24}" +procs="${SOAK_PROCS:-3}" +matches="${SOAK_MATCHES:-2}" +latency="${SOAK_LATENCY:-60}" +jitter="${SOAK_JITTER:-20}" +loss="${SOAK_LOSS:-0.02}" +seed="${SOAK_SEED:-7}" +port="${SOAK_PORT:-24680}" +hash_every="${SOAK_HASH_EVERY:-300}" +out="$repo/build/soak" +server_args=(--matches="$matches") +tamper=() +if [[ "$selfcheck" == "1" ]]; then + clients=2 + procs=1 + matches=0 + hash_every=30 + out="$repo/build/soak-selfcheck" + # Lobby and countdown take 900 ticks; then a little under a minute of the match. + server_args=(--ticks=2400) + tamper=(--tamper=1) +fi +limit_seconds=$(( matches > 0 ? matches * 1500 + 300 : 300 )) +rm -rf "$out" +mkdir -p "$out" +pids=() + +stop_all() { + for pid in "${pids[@]}"; do kill "$pid" 2>/dev/null || true; done +} +trap stop_all EXIT + +available_mb() { + awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo +} + +echo "soak: $clients clients in $procs processes, $matches matches, ${latency} ms +/- ${jitter} ms, loss $loss, seed $seed (logs in ${out#$repo/})" +nice -n 15 tools/godot.sh -- --server --listen --no-lan --port="$port" --seed="$seed" --bots=1 \ + --log-every=900 --world-hash-every="$hash_every" "${server_args[@]}" >"$out/server.log" 2>&1 & +server=$! +pids+=("$server") +sleep 3 +logs=() +per=$(( (clients + procs - 1) / procs )) +first=0 +for (( index = 0; index < procs && first < clients; index++ )); do + count=$(( clients - first < per ? clients - first : per )) + log="$out/clients_$index.log" + logs+=("$log") + nice -n 15 tools/godot.sh --script res://tools/soak_clients.gd -- --server="127.0.0.1:$port" \ + --clients="$count" --first="$first" --latency="$latency" --jitter="$jitter" --loss="$loss" \ + --seed="$seed" --hash-every="$hash_every" --seconds="$limit_seconds" "${tamper[@]}" >"$log" 2>&1 & + pids+=("$!") + first=$(( first + count )) +done + +# Wait for everything to finish, watching the host's memory and the time limit. +started=$SECONDS +low_memory=0 +while :; do + running=0 + for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && running=1; done + [[ "$running" -eq 0 ]] && break + if (( $(available_mb) < 2048 )); then + low_memory=1 + echo "soak: the host has under 2 GB available; stopping" >&2 + stop_all + break + fi + if (( SECONDS - started > limit_seconds )); then + echo "soak: still running after ${limit_seconds}s; stopping" >&2 + stop_all + break + fi + sleep 5 +done +wait 2>/dev/null +trap - EXIT +[[ "$low_memory" -eq 0 ]] || exit 1 + +joined="$(IFS=,; echo "${logs[*]}")" +report="$(tools/godot.sh --script res://tools/soak_report.gd -- --server-log="$out/server.log" \ + --client-logs="$joined" --clients="$clients" --matches="$matches" 2>&1 | grep -v '^Godot Engine')" +echo "$report" >"$out/report.txt" + +if [[ "$selfcheck" == "1" ]]; then + if grep -q "client 1's world differs" <<<"$report" && ! grep -q "client 0's world differs" <<<"$report" \ + && grep -q '"hashes_compared":[1-9]' <<<"$report"; then + echo "soak-selfcheck: ok (the tampered client was caught, the other wasn't)" + exit 0 + fi + echo "soak-selfcheck: FAIL, the tampered world wasn't caught as it should be" >&2 + echo "$report" >&2 + exit 1 +fi +echo "$report" +grep -q '^soak FAIL' <<<"$report" && exit 1 +echo "soak: ok" diff --git a/tools/soak_clients.gd b/tools/soak_clients.gd new file mode 100644 index 0000000..8dbcb46 --- /dev/null +++ b/tools/soak_clients.gd @@ -0,0 +1,276 @@ +## Runs several headless network clients in one process for the soak test (make soak). +## Each client reaches the server through its own UdpRelay, which delays, jitters and +## loses datagrams both ways, and is played by a SoakPlayer. +## +## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT +## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION] +## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N] +## +## --tamper=N knocks a point off one prop's health in client N's world just before its +## first hash once the match is under way, so `make soak-selfcheck` can prove that a +## client whose world has drifted is caught. +## +## Prints JSON lines: +## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on +## the snapshots where the server logs its own (--world-hash-every on the server, the +## same N as --hash-every here), when the client's world has caught up exactly. +## {"event":"client_match", "client", "match", ...}: a client's prediction statistics +## for a match it played, once the match is over for it. +## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began. +## {"event":"soak_client", ...}: one per client at the end. +## {"event":"soak_process", ...}: frame times, totals and errors, last. +## It stops when every client's connection has closed, or after --seconds. +## +## Making a match's island takes most of a second. A real player does it once, but a +## dozen clients here would all do it in the same frame and starve every connection of +## service for many seconds. So a MATCH notice that brings a new match is held back, +## with everything after it for that client, until the driver has made that client's +## world, and the driver makes at most one world a frame. +## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line. +extends SceneTree + +const FPS: int = 60 + +var _clients: Array[Dictionary] = [] +var _errors: ErrorCounter = ErrorCounter.new() +var _hash_every: int = 300 +var _deadline_msec: int = 0 +var _frame_usec: PackedInt64Array = PackedInt64Array() +var _memory_logged: Dictionary = {} +var _started_msec: int = 0 +var _done: bool = false +var _tamper: int = -1 + + +func _initialize() -> void: + OS.add_logger(_errors) + var options: Dictionary = { + "server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20", + "loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1", + } + for arg: String in OS.get_cmdline_user_args(): + var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1) + if parts.size() != 2 or not options.has(parts[0]): + printerr("soak_clients: unknown argument '%s'" % arg) + quit(2) + return + options[parts[0]] = parts[1] + var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT) + if target.is_empty(): + printerr("soak_clients: --server=HOST:PORT is needed") + quit(2) + return + Engine.max_fps = FPS + _hash_every = int(options["hash-every"]) + _tamper = int(options["tamper"]) + _started_msec = Time.get_ticks_msec() + _deadline_msec = _started_msec + int(options["seconds"]) * 1000 + var first: int = int(options["first"]) + for index: int in int(options["clients"]): + var number: int = first + index + var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number) + var error: Error = relay.open(0, String(target["host"]), int(target["port"])) + if error != OK: + printerr("soak_clients: couldn't open a relay: %s" % error_string(error)) + quit(2) + return + var client: Dictionary = { + "number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(), + "player": SoakPlayer.new(number), "phases": [], "frames_playing": 0, + "samples": 0, "skipped": 0, "matches": [], "stats_match": -1, + "held": [], "notice": "", "ready_world": null, "ready_seed": 0, + } + var game: ClientGame = client["game"] + var player: SoakPlayer = client["player"] + game.sample_input = func(tick: int) -> PlayerCommand: + if game.world == null: + return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0) + return player.command(tick, game.prediction.player, game.world) + game.on_snapshot = _sample.bind(client) + game.make_world = func(match_seed: int) -> Simulation: + var ready: Simulation = client["ready_world"] + client["ready_world"] = null + if ready != null and int(client["ready_seed"]) == match_seed: + return ready + return Simulation.new(match_seed) + game.attach(client["link"]) + (client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number) + _clients.append(client) + + +func _process(delta: float) -> bool: + if _done: + return true + var began: int = Time.get_ticks_usec() + var open: int = 0 + var may_make_world: bool = true + for client: Dictionary in _clients: + var relay: UdpRelay = client["relay"] + var link: NetClient = client["link"] + var game: ClientGame = client["game"] + relay.poll() + link.poll() + _take_packets(client, link.packets) + if may_make_world and not (client["held"] as Array).is_empty(): + may_make_world = false + _make_world(client) + game.advance(delta) + relay.poll() + _follow(client) + if link.state != NetClient.CLOSED: + open += 1 + _frame_usec.append(Time.get_ticks_usec() - began) + if open == 0 or Time.get_ticks_msec() >= _deadline_msec: + _finish() + return true + return false + + +## Passes packets to the client's game, holding back a MATCH notice for a new match, +## and everything after it, until its world has been made. +func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void: + var game: ClientGame = client["game"] + var held: Array = client["held"] + for packet: Dictionary in packets: + if held.is_empty() and not _brings_new_match(client, packet): + game.receive(int(packet["channel"]), packet["bytes"]) + else: + held.append(packet) + + +## True for a MATCH notice of a match (index and seed) the client hasn't been told of. +func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool: + var bytes: PackedByteArray = packet["bytes"] + if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH: + return false + var notice: Dictionary = NetSnapshot.decode_match(bytes) + if notice.is_empty(): + return false + var key: String = "%d:%d" % [notice["index"], notice["seed"]] + if key == String(client["notice"]): + return false + client["notice"] = key + client["ready_seed"] = int(notice["seed"]) + return true + + +## Makes the world for the held notice, then passes on everything held. +func _make_world(client: Dictionary) -> void: + var game: ClientGame = client["game"] + client["ready_world"] = Simulation.new(int(client["ready_seed"])) + var held: Array = client["held"] + client["held"] = [] + for packet: Dictionary in held: + game.receive(int(packet["channel"]), packet["bytes"]) + + +## Keeps track of a client's phases, and reports each match it played once it's over. +func _follow(client: Dictionary) -> void: + var game: ClientGame = client["game"] + var link: NetClient = client["link"] + if link.state == NetClient.CLOSED and not client.has("closed_s"): + # Say at once when a connection closes, with what its relay last saw each way. + var relay: UdpRelay = client["relay"] + var now: int = Time.get_ticks_msec() + client["closed_s"] = roundi((now - _started_msec) / 1000.0) + print(JSON.stringify({ + "event": "client_closed", "client": client["number"], "reason": link.close_reason, + "at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index, + "held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(), + "since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1, + "since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1, + })) + if game.world == null: + return + var phases: Array = client["phases"] + if phases.is_empty() or phases[phases.size() - 1] != game.match_phase: + phases.append(game.match_phase) + if game.playing(): + client["frames_playing"] = int(client["frames_playing"]) + 1 + client["stats_match"] = game.match_index + client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction] + elif int(client["stats_match"]) >= 0: + _report_match(client) + # Once a match is in play every client here has let go of the last match's world. + if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index): + _memory_logged[game.match_index] = true + print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage()})) + + +func _report_match(client: Dictionary) -> void: + var stats: Array = client["last_stats"] + var record: Dictionary = { + "event": "client_match", "client": client["number"], "match": client["stats_match"], + "reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01), + } + (client["matches"] as Array).append(record) + print(JSON.stringify(record)) + client["stats_match"] = -1 + + +## After each snapshot: on the server's sampling ticks, the client's world hash, if its +## world has caught up to exactly what the snapshot says the server had sent. +func _sample(client: Dictionary) -> void: + var game: ClientGame = client["game"] + var tick: int = game.receiver.sim_tick + # Tick 0 lasts the whole lobby and countdown while the world is still being set up, + # so it names more than one state; the server doesn't log it either. + if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS: + return + if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0: + client["skipped"] = int(client["skipped"]) + 1 + return + client["samples"] = int(client["samples"]) + 1 + if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0: + _tamper = -1 + game.world.props.health[0] -= 1 + print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)})) + + +func _finish() -> void: + _done = true + var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0} + for client: Dictionary in _clients: + var relay: UdpRelay = client["relay"] + var link: NetClient = client["link"] + var game: ClientGame = client["game"] + if int(client["stats_match"]) >= 0: + _report_match(client) + var phase_names: Array[String] = [] + for phase: int in client["phases"]: + phase_names.append(Match.PHASE_NAMES[phase]) + print(JSON.stringify({ + "event": "soak_client", "client": client["number"], "name": link.player_name, + "joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names, + "frames_playing": client["frames_playing"], "bad_packets": game.bad_packets, + "samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(), + "build_attempts": (client["player"] as SoakPlayer).build_attempts, + "pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back, + "world_gaps": game.replica.gaps, + "down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes, + })) + for key: String in totals: + totals[key] = int(totals[key]) + int(relay.get(key)) + link.leave() + relay.close() + # The input and snapshot callables hold the client: let go of them, or nothing frees. + game.sample_input = Callable() + game.on_snapshot = Callable() + game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed) + client["ready_world"] = null + var sorted: PackedInt64Array = _frame_usec.duplicate() + sorted.sort() + var summary: Dictionary = { + "event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0), + "frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(), + } + if not sorted.is_empty(): + summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50) + summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99) + summary.merge(totals) + if _errors.count() > 0: + summary["first_error"] = _errors.first() + print(JSON.stringify(summary)) + _clients.clear() + OS.remove_logger(_errors) + quit(1 if _errors.count() > 0 else 0) diff --git a/tools/soak_clients.gd.uid b/tools/soak_clients.gd.uid new file mode 100644 index 0000000..14f6892 --- /dev/null +++ b/tools/soak_clients.gd.uid @@ -0,0 +1 @@ +uid://gsjp33cla4sm diff --git a/tools/soak_report.gd b/tools/soak_report.gd new file mode 100644 index 0000000..17d3cc6 --- /dev/null +++ b/tools/soak_report.gd @@ -0,0 +1,227 @@ +## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the +## server ran its matches cleanly and within its tick budget, every client joined and +## played every match with no unreadable packets, every client's copy of the world +## matched the server's wherever both took a hash, and neither the server nor the +## client processes grew in memory from one match to the next. +## +## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH +## --client-logs=PATH[,PATH...] --clients=N --matches=N +## +## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own +## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line. +extends SceneTree + +## How much a process's memory may grow from the first match to the last before it +## counts as a leak. Each match makes a new world, so some wobble is normal. +const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024 +## Hash comparisons each client must reach per match, so a run that compared nothing +## can't pass. +const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10 + +var _failures: Array[String] = [] + + +func _initialize() -> void: + var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"} + for arg: String in OS.get_cmdline_user_args(): + var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1) + if parts.size() != 2 or not options.has(parts[0]): + printerr("soak_report: unknown argument '%s'" % arg) + quit(2) + return + options[parts[0]] = parts[1] + if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty(): + printerr("soak_report: --server-log and --client-logs are needed") + quit(2) + return + var result: Dictionary = check( + _read(String(options["server-log"])), + _read_all(String(options["client-logs"]).split(",")), + int(options["clients"]), int(options["matches"])) + print(JSON.stringify(result)) + for failure: String in _failures: + print("soak FAIL: " + failure) + quit(0 if _failures.is_empty() else 1) + + +## Checks a run from its parsed log lines. Returns the measurements; failures go in +## `_failures`. +func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary: + var result: Dictionary = {"event": "soak"} + # The server. + var stats: Dictionary = _first(server, "stats") + var stop: Dictionary = _first(server, "stop") + if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0: + _fail("the server didn't stop cleanly: %s" % JSON.stringify(stop)) + var ended: int = _count(server, "results") + result["matches_ended"] = ended + if ended < matches: + _fail("the server ended %d matches, not %d" % [ended, matches]) + var budget: int = int(stats.get("budget_us", 0)) + result["tick_p99_us"] = stats.get("with_net_p99_us", -1) + result["tick_max_us"] = stats.get("with_net_max_us", -1) + result["sim_p99_us"] = stats.get("p99_us", -1) + result["over_budget_ticks"] = stats.get("over_budget", -1) + if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget: + _fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget]) + result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server") + # The clients. + var reports: Array[Dictionary] = _all(clients, "soak_client") + result["clients"] = reports.size() + if reports.size() != client_count: + _fail("%d clients reported, not %d" % [reports.size(), client_count]) + var down: int = 0 + var up: int = 0 + var played: Dictionary = {} + for record: Dictionary in _all(clients, "client_match"): + played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1 + for report: Dictionary in reports: + var number: int = int(report["client"]) + down += int(report["down_bytes"]) + up += int(report["up_bytes"]) + if not report["joined"]: + _fail("client %d never joined" % number) + if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN: + _fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)]) + if int(report["bad_packets"]) != 0: + _fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]]) + if int(played.get(number, 0)) < matches: + _fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches]) + var processes: Array[Dictionary] = _all(clients, "soak_process") + var seconds: int = 0 + var mispredictions: int = 0 + var reconciles: int = 0 + var snaps: int = 0 + for record: Dictionary in _all(clients, "client_match"): + mispredictions += int(record["mispredictions"]) + reconciles += int(record["reconciles"]) + snaps += int(record["snaps"]) + var lost: int = 0 + var sent: int = 0 + for process: Dictionary in processes: + seconds = maxi(seconds, int(process["seconds"])) + lost += int(process["up_lost"]) + int(process["down_lost"]) + sent += int(process["up_packets"]) + int(process["down_packets"]) + if int(process["errors"]) != 0: + _fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")]) + result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0))) + result["client_processes"] = processes.size() + result["seconds"] = seconds + result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001) + if seconds > 0 and not reports.is_empty(): + result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1) + result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1) + result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1) + result["reconciles"] = reconciles + result["mispredictions"] = mispredictions + result["prediction_snaps"] = snaps + result["client_mem_growth_bytes"] = _client_growth(clients) + # The world, compared hash by hash. + var server_hashes: Dictionary = {} + for record: Dictionary in _all(server, "world_hash"): + server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"]) + var compared: int = 0 + var differ: int = 0 + var per_client: Dictionary = {} + for record: Dictionary in _all(clients, "hash"): + var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"])) + if not server_hashes.has(key): + continue + compared += 1 + per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1 + if server_hashes[key] != String(record["hash"]): + differ += 1 + if differ <= 5: + _fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y]) + result["hashes_compared"] = compared + result["hashes_differing"] = differ + if differ > 5: + _fail("%d world hashes differed in all" % differ) + for report: Dictionary in reports: + var number: int = int(report["client"]) + if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches: + _fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)]) + return result + + +## How much the server's memory grew from the first `phase` entry it logged to the last. +func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int: + var samples: Array[int] = [] + for record: Dictionary in _all(lines, event): + if String(record.get("to", "")) == phase and record.has("mem_bytes"): + samples.append(int(record["mem_bytes"])) + return _judge(samples, who) + + +## The same for each client process, from its "memory" lines. +func _client_growth(lines: Array[Dictionary]) -> int: + var by_process: Dictionary = {} + for record: Dictionary in lines: + if String(record.get("event", "")) == "memory": + var process: int = int(record.get("_process", 0)) + if not by_process.has(process): + by_process[process] = [] as Array[int] + (by_process[process] as Array[int]).append(int(record["mem_bytes"])) + var worst: int = 0 + for process: int in by_process: + worst = maxi(worst, _judge(by_process[process], "client process %d" % process)) + return worst + + +func _judge(samples: Array[int], who: String) -> int: + if samples.size() < 2: + _fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()]) + return 0 + var growth: int = samples[samples.size() - 1] - samples[0] + if growth > LEAK_ALLOWANCE_BYTES: + _fail("%s grew by %d bytes from the first match to the last" % [who, growth]) + return growth + + +func _fail(message: String) -> void: + _failures.append(message) + + +func _read_all(paths: PackedStringArray) -> Array[Dictionary]: + var lines: Array[Dictionary] = [] + for index: int in paths.size(): + for record: Dictionary in _read(paths[index]): + record["_process"] = index + lines.append(record) + return lines + + +## The JSON object lines of a log; anything else in it is skipped. +func _read(path: String) -> Array[Dictionary]: + var lines: Array[Dictionary] = [] + var file: FileAccess = FileAccess.open(path, FileAccess.READ) + if file == null: + _fail("couldn't read %s" % path) + return lines + while not file.eof_reached(): + var line: String = file.get_line() + if not line.begins_with("{"): + continue + var parsed: Variant = JSON.parse_string(line) + if parsed is Dictionary: + lines.append(parsed) + return lines + + +func _first(lines: Array[Dictionary], event: String) -> Dictionary: + for record: Dictionary in lines: + if String(record.get("event", "")) == event: + return record + return {} + + +func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]: + var found: Array[Dictionary] = [] + for record: Dictionary in lines: + if String(record.get("event", "")) == event: + found.append(record) + return found + + +func _count(lines: Array[Dictionary], event: String) -> int: + return _all(lines, event).size() diff --git a/tools/soak_report.gd.uid b/tools/soak_report.gd.uid new file mode 100644 index 0000000..65416ed --- /dev/null +++ b/tools/soak_report.gd.uid @@ -0,0 +1 @@ +uid://byjci1haeqpso