H4: Long-run stability (make stability), and two memory leaks fixed
make stability (tools/stability.sh, StabilityReport) runs one dedicated server for 50 bot matches back to back, unthrottled, sampling its resident memory, and fails on a crash, hang, engine or script error, a match with no winner, or memory that grows after the first five matches. The first run had no crash, hang or error, but showed two leaks: - ServerLoop kept every tick's duration in an array (8 bytes a tick, about 21 MB a day at 30 Hz) and copied and sorted all of it for every progress line. TickTimes keeps exact statistics in fixed memory: a one-microsecond histogram up to 65 ms, plus the rare slower ticks. - Resident memory grew about 28 MB a match (905 MB after 42 matches): glibc gives each thread its own malloc arena, and every next island is built on a worker thread. The server launchers (packaging/server, packaging/dist, so also the .deb, Docker image and systemd unit) and tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625). After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no errors; static memory +1.6 KiB and resident +145 KiB a match, flat at 204 MB from match 26 on. No match crashed, so there is no crash seed to turn into a regression test. Tests: TickTimes against a sorted list, its fixed memory, the launchers' setting, and StabilityReport. make check, dist-check and server-package-check pass. Closes #57 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -34,7 +34,7 @@ var _finished: bool = false
|
||||
var _matches_ended: int = 0
|
||||
## When listening: microseconds each tick took including the network's share (after_tick:
|
||||
## world messages and snapshots), which ServerLoop's own timing leaves out.
|
||||
var _net_tick_usec: PackedInt64Array = PackedInt64Array()
|
||||
var _net_tick_usec: TickTimes = TickTimes.new()
|
||||
|
||||
|
||||
func _init(server_config: ServerConfig) -> void:
|
||||
@@ -191,7 +191,7 @@ func _tick() -> void:
|
||||
loop.run_tick()
|
||||
if game != null:
|
||||
game.after_tick()
|
||||
_net_tick_usec.append(Time.get_ticks_usec() - began)
|
||||
_net_tick_usec.add(Time.get_ticks_usec() - began)
|
||||
if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
||||
_log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)})
|
||||
for event: Dictionary in loop.match_flow.events:
|
||||
@@ -223,12 +223,10 @@ func _finish(requested_code: int) -> void:
|
||||
var stats: Dictionary = loop.timing_stats()
|
||||
stats["state_hash"] = loop.sim.state_hash()
|
||||
stats["entities"] = loop.sim.entities.count()
|
||||
if not _net_tick_usec.is_empty():
|
||||
var sorted: PackedInt64Array = _net_tick_usec.duplicate()
|
||||
sorted.sort()
|
||||
stats["with_net_p50_us"] = ServerLoop._percentile(sorted, 0.50)
|
||||
stats["with_net_p99_us"] = ServerLoop._percentile(sorted, 0.99)
|
||||
stats["with_net_max_us"] = sorted[sorted.size() - 1]
|
||||
if _net_tick_usec.size() > 0:
|
||||
stats["with_net_p50_us"] = _net_tick_usec.percentile(0.50)
|
||||
stats["with_net_p99_us"] = _net_tick_usec.percentile(0.99)
|
||||
stats["with_net_max_us"] = _net_tick_usec.stats(ServerLoop.TICK_BUDGET_USEC)["max_us"]
|
||||
stats["mem_bytes"] = OS.get_static_memory_usage()
|
||||
_log("stats", stats)
|
||||
if code == 0 and _errors.count() > 0:
|
||||
|
||||
@@ -24,8 +24,8 @@ var nav: NavGrid = null
|
||||
var world: BotWorld = null
|
||||
## Every bot's difficulty, or -1 to mix them: easy, normal, hard, in turn.
|
||||
var bot_difficulty: int = -1
|
||||
## Wall-clock duration of every tick run, in microseconds.
|
||||
var tick_usec: PackedInt64Array = PackedInt64Array()
|
||||
## Wall-clock duration of every tick run, in microseconds, in fixed memory.
|
||||
var tick_times: TickTimes = TickTimes.new()
|
||||
## Server ticks run, across every match.
|
||||
var ticks: int = 0
|
||||
|
||||
@@ -102,7 +102,7 @@ func run_tick() -> void:
|
||||
submit(match_flow.handle_of[bot.player_id], bot.think(sim, nav, world))
|
||||
_advance()
|
||||
ticks += 1
|
||||
tick_usec.append(Time.get_ticks_usec() - started)
|
||||
tick_times.add(Time.get_ticks_usec() - started)
|
||||
|
||||
|
||||
func _advance() -> void:
|
||||
@@ -150,28 +150,7 @@ func _track_players() -> void:
|
||||
|
||||
## Timing statistics over every tick run so far, in microseconds.
|
||||
func timing_stats() -> Dictionary:
|
||||
var count: int = tick_usec.size()
|
||||
if count == 0:
|
||||
return {"ticks": 0}
|
||||
var sorted: PackedInt64Array = tick_usec.duplicate()
|
||||
sorted.sort()
|
||||
var total: int = 0
|
||||
var over_budget: int = 0
|
||||
for value: int in sorted:
|
||||
total += value
|
||||
if value > TICK_BUDGET_USEC:
|
||||
over_budget += 1
|
||||
return {
|
||||
"ticks": count,
|
||||
"min_us": sorted[0],
|
||||
"mean_us": roundi(float(total) / count),
|
||||
"p50_us": _percentile(sorted, 0.50),
|
||||
"p95_us": _percentile(sorted, 0.95),
|
||||
"p99_us": _percentile(sorted, 0.99),
|
||||
"max_us": sorted[count - 1],
|
||||
"budget_us": TICK_BUDGET_USEC,
|
||||
"over_budget": over_budget,
|
||||
}
|
||||
return tick_times.stats(TICK_BUDGET_USEC)
|
||||
|
||||
|
||||
## Nearest-rank percentile of an ascending array.
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
## Tick durations in microseconds, kept in fixed memory however long the server runs
|
||||
## (H4: a list of every tick grew by 8 bytes a tick, about 21 MB a day, and was copied
|
||||
## and sorted for every progress line). A one-microsecond histogram up to HISTOGRAM_USEC
|
||||
## makes every statistic exact below that; the few ticks above it (a lobby waiting on
|
||||
## the next island, say) are kept as they are, up to MAX_SLOW of them.
|
||||
class_name TickTimes
|
||||
extends RefCounted
|
||||
|
||||
const HISTOGRAM_USEC: int = 65536
|
||||
const MAX_SLOW: int = 4096
|
||||
|
||||
var _counts: PackedInt32Array = PackedInt32Array()
|
||||
## Ticks at or over HISTOGRAM_USEC, ascending once sorted.
|
||||
var _slow: PackedInt64Array = PackedInt64Array()
|
||||
var _slow_sorted: bool = true
|
||||
var _slow_dropped: int = 0
|
||||
var _count: int = 0
|
||||
var _total: int = 0
|
||||
var _min: int = 0
|
||||
var _max: int = 0
|
||||
|
||||
|
||||
func _init() -> void:
|
||||
_counts.resize(HISTOGRAM_USEC)
|
||||
|
||||
|
||||
func add(usec: int) -> void:
|
||||
usec = maxi(usec, 0)
|
||||
if _count == 0:
|
||||
_min = usec
|
||||
_max = usec
|
||||
_min = mini(_min, usec)
|
||||
_max = maxi(_max, usec)
|
||||
_count += 1
|
||||
_total += usec
|
||||
if usec < HISTOGRAM_USEC:
|
||||
_counts[usec] += 1
|
||||
elif _slow.size() < MAX_SLOW:
|
||||
_slow.append(usec)
|
||||
_slow_sorted = false
|
||||
else:
|
||||
# Past the cap, keep the slowest: drop the fastest slow tick if this one is slower.
|
||||
_sort_slow()
|
||||
_slow_dropped += 1
|
||||
if usec > _slow[0]:
|
||||
_slow[0] = usec
|
||||
_slow_sorted = false
|
||||
|
||||
|
||||
func size() -> int:
|
||||
return _count
|
||||
|
||||
|
||||
## Nearest-rank percentile, as ServerLoop._percentile gives over a sorted list.
|
||||
func percentile(fraction: float) -> int:
|
||||
if _count == 0:
|
||||
return 0
|
||||
var rank: int = clampi(ceili(fraction * _count) - 1, 0, _count - 1)
|
||||
var seen: int = 0
|
||||
for usec: int in HISTOGRAM_USEC:
|
||||
seen += _counts[usec]
|
||||
if seen > rank:
|
||||
return usec
|
||||
_sort_slow()
|
||||
# Ranks among the slow ticks; any dropped past the cap were the fastest of them.
|
||||
var index: int = rank - seen - _slow_dropped
|
||||
return _slow[clampi(index, 0, _slow.size() - 1)] if not _slow.is_empty() else _max
|
||||
|
||||
|
||||
## {"ticks", "min_us", "mean_us", "p50_us", "p95_us", "p99_us", "max_us", "budget_us",
|
||||
## "over_budget"}, or {"ticks": 0} before the first tick.
|
||||
func stats(budget_usec: int) -> Dictionary:
|
||||
if _count == 0:
|
||||
return {"ticks": 0}
|
||||
return {
|
||||
"ticks": _count,
|
||||
"min_us": _min,
|
||||
"mean_us": roundi(float(_total) / _count),
|
||||
"p50_us": percentile(0.50),
|
||||
"p95_us": percentile(0.95),
|
||||
"p99_us": percentile(0.99),
|
||||
"max_us": _max,
|
||||
"budget_us": budget_usec,
|
||||
"over_budget": over(budget_usec),
|
||||
}
|
||||
|
||||
|
||||
## How many ticks took longer than `budget_usec`.
|
||||
func over(budget_usec: int) -> int:
|
||||
var count: int = 0
|
||||
if budget_usec < HISTOGRAM_USEC:
|
||||
for usec: int in range(budget_usec + 1, HISTOGRAM_USEC):
|
||||
count += _counts[usec]
|
||||
for usec: int in _slow:
|
||||
if usec > budget_usec:
|
||||
count += 1
|
||||
return count + _slow_dropped
|
||||
|
||||
|
||||
func _sort_slow() -> void:
|
||||
if not _slow_sorted:
|
||||
_slow.sort()
|
||||
_slow_sorted = true
|
||||
@@ -0,0 +1 @@
|
||||
uid://f8axsl0jjws6
|
||||
Reference in New Issue
Block a user