diff --git a/tools/soak_clients.gd b/tools/soak_clients.gd index 8f49cbb..c36f945 100644 --- a/tools/soak_clients.gd +++ b/tools/soak_clients.gd @@ -28,6 +28,7 @@ extends SceneTree const FPS: int = 60 +const LATE_MEMORY_MSEC: int = 60000 var _clients: Array[Dictionary] = [] var _errors: ErrorCounter = ErrorCounter.new() @@ -115,6 +116,21 @@ func _process(delta: float) -> bool: return false +## How many distinct worlds this process's clients still reach: their own, and the +## ground their prediction runs on. +func _live_worlds() -> Dictionary: + var worlds: Dictionary = {} + var grounds: Dictionary = {} + for client: Dictionary in _clients: + var game: ClientGame = client["game"] + if game.world != null: + worlds[game.world.get_instance_id()] = true + var ground: SimGround = game.prediction.get("_ground") + if ground != null: + grounds[ground.get_instance_id()] = true + return {"worlds": worlds.size(), "prediction_grounds": grounds.size()} + + ## Keeps track of a client's phases, and reports each match it played once it's over. func _follow(client: Dictionary) -> void: var game: ClientGame = client["game"] @@ -144,8 +160,13 @@ func _follow(client: Dictionary) -> void: _report_match(client) # Once a match is in play every client here has let go of the last match's world. if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index): - _memory_logged[game.match_index] = true - print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage()})) + _memory_logged[game.match_index] = Time.get_ticks_msec() + print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()})) + elif game.match_phase == Match.PLAY and _memory_logged.has(game.match_index) and _memory_logged.has(-1 - game.match_index) == false \ + and Time.get_ticks_msec() - int(_memory_logged[game.match_index]) > LATE_MEMORY_MSEC: + # A second look a minute later, for telling a leak from something let go late. + _memory_logged[-1 - game.match_index] = true + print(JSON.stringify({"event": "memory_late", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage(), "objects": Performance.get_monitor(Performance.OBJECT_COUNT), "worlds": _live_worlds()})) func _report_match(client: Dictionary) -> void: