## Runs several headless network clients in one process for the soak test (make soak). ## Each client reaches the server through its own UdpRelay, which delays, jitters and ## loses datagrams both ways, and is played by a SoakPlayer. ## ## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT ## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION] ## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N] ## ## --tamper=N knocks a point off one prop's health in client N's world just before its ## first hash once the match is under way, so `make soak-selfcheck` can prove that a ## client whose world has drifted is caught. ## ## Prints JSON lines: ## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on ## the snapshots where the server logs its own (--world-hash-every on the server, the ## same N as --hash-every here), when the client's world has caught up exactly. ## {"event":"client_match", "client", "match", ...}: a client's prediction statistics ## for a match it played, once the match is over for it. ## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began. ## {"event":"soak_client", ...}: one per client at the end. ## {"event":"soak_process", ...}: frame times, totals and errors, last. ## It stops when every client's connection has closed, or after --seconds. ## ## Making a match's island takes most of a second. Each client makes it on a worker ## thread (ClientGame.build_in_background), as the real client does, so the frame loop ## keeps servicing every connection while islands are made. ## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line. extends SceneTree const FPS: int = 60 const LATE_MEMORY_MSEC: int = 60000 var _clients: Array[Dictionary] = [] var _errors: ErrorCounter = ErrorCounter.new() var _hash_every: int = 300 var _deadline_msec: int = 0 var _frame_usec: PackedInt64Array = PackedInt64Array() var _memory_logged: Dictionary = {} var _started_msec: int = 0 var _done: bool = false var _tamper: int = -1 func _initialize() -> void: OS.add_logger(_errors) var options: Dictionary = { "server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20", "loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1", } for arg: String in OS.get_cmdline_user_args(): var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1) if parts.size() != 2 or not options.has(parts[0]): printerr("soak_clients: unknown argument '%s'" % arg) quit(2) return options[parts[0]] = parts[1] var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT) if target.is_empty(): printerr("soak_clients: --server=HOST:PORT is needed") quit(2) return Engine.max_fps = FPS _hash_every = int(options["hash-every"]) _tamper = int(options["tamper"]) _started_msec = Time.get_ticks_msec() _deadline_msec = _started_msec + int(options["seconds"]) * 1000 var first: int = int(options["first"]) for index: int in int(options["clients"]): var number: int = first + index var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number) var error: Error = relay.open(0, String(target["host"]), int(target["port"])) if error != OK: printerr("soak_clients: couldn't open a relay: %s" % error_string(error)) quit(2) return var client: Dictionary = { "number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(), "player": SoakPlayer.new(number), "phases": [], "frames_playing": 0, "samples": 0, "skipped": 0, "matches": [], "stats_match": -1, } var game: ClientGame = client["game"] var player: SoakPlayer = client["player"] game.sample_input = func(tick: int) -> PlayerCommand: if game.world == null: return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0) return player.command(tick, game.prediction.player, game.world) game.on_snapshot = _sample.bind(client) game.build_in_background = true game.attach(client["link"]) (client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number) _clients.append(client) func _process(delta: float) -> bool: if _done: return true var began: int = Time.get_ticks_usec() var open: int = 0 for client: Dictionary in _clients: var relay: UdpRelay = client["relay"] var link: NetClient = client["link"] var game: ClientGame = client["game"] relay.poll() link.poll() for packet: Dictionary in link.packets: game.receive(int(packet["channel"]), packet["bytes"]) game.advance(delta) relay.poll() _follow(client) if link.state != NetClient.CLOSED: open += 1 _frame_usec.append(Time.get_ticks_usec() - began) if open == 0 or Time.get_ticks_msec() >= _deadline_msec: _finish() return true return false ## How many distinct worlds this process's clients still reach (their own, the ground ## their prediction runs on, and islands made in the background but not yet taken ## up), and how many Simulations and grounds are alive in all. `extra_islands` counts ## live Simulations nothing here accounts for: a leak (#39). func _live_worlds() -> Dictionary: var worlds: Dictionary = {} var grounds: Dictionary = {} var built: int = 0 for client: Dictionary in _clients: var game: ClientGame = client["game"] if game.world != null: worlds[game.world.get_instance_id()] = true var ground: SimGround = game.prediction.get("_ground") if ground != null: grounds[ground.get_instance_id()] = true var made: Simulation = game.get("_built") if made != null and made != game.world: built += 1 if game.building_world(): built += 1 var live: int = LiveCount.of(&"Simulation") return {"worlds": worlds.size(), "prediction_grounds": grounds.size(), "built": built, "live_simulations": live, "live_grounds": LiveCount.of(&"SimGround"), "extra_islands": maxi(0, live - worlds.size() - built)} ## Keeps track of a client's phases, and reports each match it played once it's over. func _follow(client: Dictionary) -> void: var game: ClientGame = client["game"] var link: NetClient = client["link"] if link.state == NetClient.CLOSED and not client.has("closed_s"): # Say at once when a connection closes, with what its relay last saw each way. var relay: UdpRelay = client["relay"] var now: int = Time.get_ticks_msec() client["closed_s"] = roundi((now - _started_msec) / 1000.0) print(JSON.stringify({ "event": "client_closed", "client": client["number"], "reason": link.close_reason, "at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index, "held": game.held_count(), "relay_in_flight": relay.in_flight(), "since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1, "since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1, })) if game.world == null: return var phases: Array = client["phases"] if phases.is_empty() or phases[phases.size() - 1] != game.match_phase: phases.append(game.match_phase) if game.playing(): client["frames_playing"] = int(client["frames_playing"]) + 1 client["stats_match"] = game.match_index client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction] elif int(client["stats_match"]) >= 0: _report_match(client) # Once a match is in play every client here has let go of the last match's world. if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index): _memory_logged[game.match_index] = Time.get_ticks_msec() print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()})) elif game.match_phase == Match.PLAY and _memory_logged.has(game.match_index) and _memory_logged.has(-1 - game.match_index) == false \ and Time.get_ticks_msec() - int(_memory_logged[game.match_index]) > LATE_MEMORY_MSEC: # A second look a minute later, for telling a leak from something let go late. _memory_logged[-1 - game.match_index] = true print(JSON.stringify({"event": "memory_late", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()})) func _report_match(client: Dictionary) -> void: var stats: Array = client["last_stats"] var record: Dictionary = { "event": "client_match", "client": client["number"], "match": client["stats_match"], "reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01), } (client["matches"] as Array).append(record) print(JSON.stringify(record)) client["stats_match"] = -1 ## After each snapshot: on the server's sampling ticks, the client's world hash, if its ## world has caught up to exactly what the snapshot says the server had sent. func _sample(client: Dictionary) -> void: var game: ClientGame = client["game"] var tick: int = game.receiver.sim_tick # Tick 0 lasts the whole lobby and countdown while the world is still being set up, # so it names more than one state; the server doesn't log it either. if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS: return if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0: client["skipped"] = int(client["skipped"]) + 1 return client["samples"] = int(client["samples"]) + 1 if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0: _tamper = -1 game.world.props.health[0] -= 1 print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)})) func _finish() -> void: _done = true var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0} for client: Dictionary in _clients: var relay: UdpRelay = client["relay"] var link: NetClient = client["link"] var game: ClientGame = client["game"] if int(client["stats_match"]) >= 0: _report_match(client) var phase_names: Array[String] = [] for phase: int in client["phases"]: phase_names.append(Match.PHASE_NAMES[phase]) print(JSON.stringify({ "event": "soak_client", "client": client["number"], "name": link.player_name, "joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names, "frames_playing": client["frames_playing"], "bad_packets": game.bad_packets, "samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(), "build_attempts": (client["player"] as SoakPlayer).build_attempts, "pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back, "world_gaps": game.replica.gaps, "down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes, })) for key: String in totals: totals[key] = int(totals[key]) + int(relay.get(key)) link.leave() relay.close() # The input and snapshot callables hold the client: let go of them, or nothing frees. game.sample_input = Callable() game.on_snapshot = Callable() var sorted: PackedInt64Array = _frame_usec.duplicate() sorted.sort() var summary: Dictionary = { "event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0), "frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(), } if not sorted.is_empty(): summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50) summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99) summary.merge(totals) if _errors.count() > 0: summary["first_error"] = _errors.first() print(JSON.stringify(summary)) _clients.clear() OS.remove_logger(_errors) quit(1 if _errors.count() > 0 else 0)