H4: Long-run stability (make stability), and two memory leaks fixed

make stability (tools/stability.sh, StabilityReport) runs one dedicated
server for 50 bot matches back to back, unthrottled, sampling its resident
memory, and fails on a crash, hang, engine or script error, a match with
no winner, or memory that grows after the first five matches.

The first run had no crash, hang or error, but showed two leaks:
- ServerLoop kept every tick's duration in an array (8 bytes a tick,
  about 21 MB a day at 30 Hz) and copied and sorted all of it for every
  progress line. TickTimes keeps exact statistics in fixed memory: a
  one-microsecond histogram up to 65 ms, plus the rare slower ticks.
- Resident memory grew about 28 MB a match (905 MB after 42 matches):
  glibc gives each thread its own malloc arena, and every next island is
  built on a worker thread. The server launchers (packaging/server,
  packaging/dist, so also the .deb, Docker image and systemd unit) and
  tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already
  set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625).

After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no
errors; static memory +1.6 KiB and resident +145 KiB a match, flat at
204 MB from match 26 on. No match crashed, so there is no crash seed to
turn into a regression test.

Tests: TickTimes against a sorted list, its fixed memory, the launchers'
setting, and StabilityReport. make check, dist-check and
server-package-check pass.

Closes #57

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-11-14 07:15:23 -05:00
co-authored by Claude Opus 5.5
parent 38812120df
commit f87d5eae0d
19 changed files with 486 additions and 34 deletions
+6 -8
View File
@@ -34,7 +34,7 @@ var _finished: bool = false
var _matches_ended: int = 0
## When listening: microseconds each tick took including the network's share (after_tick:
## world messages and snapshots), which ServerLoop's own timing leaves out.
var _net_tick_usec: PackedInt64Array = PackedInt64Array()
var _net_tick_usec: TickTimes = TickTimes.new()
func _init(server_config: ServerConfig) -> void:
@@ -191,7 +191,7 @@ func _tick() -> void:
loop.run_tick()
if game != null:
game.after_tick()
_net_tick_usec.append(Time.get_ticks_usec() - began)
_net_tick_usec.add(Time.get_ticks_usec() - began)
if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
_log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)})
for event: Dictionary in loop.match_flow.events:
@@ -223,12 +223,10 @@ func _finish(requested_code: int) -> void:
var stats: Dictionary = loop.timing_stats()
stats["state_hash"] = loop.sim.state_hash()
stats["entities"] = loop.sim.entities.count()
if not _net_tick_usec.is_empty():
var sorted: PackedInt64Array = _net_tick_usec.duplicate()
sorted.sort()
stats["with_net_p50_us"] = ServerLoop._percentile(sorted, 0.50)
stats["with_net_p99_us"] = ServerLoop._percentile(sorted, 0.99)
stats["with_net_max_us"] = sorted[sorted.size() - 1]
if _net_tick_usec.size() > 0:
stats["with_net_p50_us"] = _net_tick_usec.percentile(0.50)
stats["with_net_p99_us"] = _net_tick_usec.percentile(0.99)
stats["with_net_max_us"] = _net_tick_usec.stats(ServerLoop.TICK_BUDGET_USEC)["max_us"]
stats["mem_bytes"] = OS.get_static_memory_usage()
_log("stats", stats)
if code == 0 and _errors.count() > 0:
+4 -25
View File
@@ -24,8 +24,8 @@ var nav: NavGrid = null
var world: BotWorld = null
## Every bot's difficulty, or -1 to mix them: easy, normal, hard, in turn.
var bot_difficulty: int = -1
## Wall-clock duration of every tick run, in microseconds.
var tick_usec: PackedInt64Array = PackedInt64Array()
## Wall-clock duration of every tick run, in microseconds, in fixed memory.
var tick_times: TickTimes = TickTimes.new()
## Server ticks run, across every match.
var ticks: int = 0
@@ -102,7 +102,7 @@ func run_tick() -> void:
submit(match_flow.handle_of[bot.player_id], bot.think(sim, nav, world))
_advance()
ticks += 1
tick_usec.append(Time.get_ticks_usec() - started)
tick_times.add(Time.get_ticks_usec() - started)
func _advance() -> void:
@@ -150,28 +150,7 @@ func _track_players() -> void:
## Timing statistics over every tick run so far, in microseconds.
func timing_stats() -> Dictionary:
var count: int = tick_usec.size()
if count == 0:
return {"ticks": 0}
var sorted: PackedInt64Array = tick_usec.duplicate()
sorted.sort()
var total: int = 0
var over_budget: int = 0
for value: int in sorted:
total += value
if value > TICK_BUDGET_USEC:
over_budget += 1
return {
"ticks": count,
"min_us": sorted[0],
"mean_us": roundi(float(total) / count),
"p50_us": _percentile(sorted, 0.50),
"p95_us": _percentile(sorted, 0.95),
"p99_us": _percentile(sorted, 0.99),
"max_us": sorted[count - 1],
"budget_us": TICK_BUDGET_USEC,
"over_budget": over_budget,
}
return tick_times.stats(TICK_BUDGET_USEC)
## Nearest-rank percentile of an ascending array.
+103
View File
@@ -0,0 +1,103 @@
## Tick durations in microseconds, kept in fixed memory however long the server runs
## (H4: a list of every tick grew by 8 bytes a tick, about 21 MB a day, and was copied
## and sorted for every progress line). A one-microsecond histogram up to HISTOGRAM_USEC
## makes every statistic exact below that; the few ticks above it (a lobby waiting on
## the next island, say) are kept as they are, up to MAX_SLOW of them.
class_name TickTimes
extends RefCounted
const HISTOGRAM_USEC: int = 65536
const MAX_SLOW: int = 4096
var _counts: PackedInt32Array = PackedInt32Array()
## Ticks at or over HISTOGRAM_USEC, ascending once sorted.
var _slow: PackedInt64Array = PackedInt64Array()
var _slow_sorted: bool = true
var _slow_dropped: int = 0
var _count: int = 0
var _total: int = 0
var _min: int = 0
var _max: int = 0
func _init() -> void:
_counts.resize(HISTOGRAM_USEC)
func add(usec: int) -> void:
usec = maxi(usec, 0)
if _count == 0:
_min = usec
_max = usec
_min = mini(_min, usec)
_max = maxi(_max, usec)
_count += 1
_total += usec
if usec < HISTOGRAM_USEC:
_counts[usec] += 1
elif _slow.size() < MAX_SLOW:
_slow.append(usec)
_slow_sorted = false
else:
# Past the cap, keep the slowest: drop the fastest slow tick if this one is slower.
_sort_slow()
_slow_dropped += 1
if usec > _slow[0]:
_slow[0] = usec
_slow_sorted = false
func size() -> int:
return _count
## Nearest-rank percentile, as ServerLoop._percentile gives over a sorted list.
func percentile(fraction: float) -> int:
if _count == 0:
return 0
var rank: int = clampi(ceili(fraction * _count) - 1, 0, _count - 1)
var seen: int = 0
for usec: int in HISTOGRAM_USEC:
seen += _counts[usec]
if seen > rank:
return usec
_sort_slow()
# Ranks among the slow ticks; any dropped past the cap were the fastest of them.
var index: int = rank - seen - _slow_dropped
return _slow[clampi(index, 0, _slow.size() - 1)] if not _slow.is_empty() else _max
## {"ticks", "min_us", "mean_us", "p50_us", "p95_us", "p99_us", "max_us", "budget_us",
## "over_budget"}, or {"ticks": 0} before the first tick.
func stats(budget_usec: int) -> Dictionary:
if _count == 0:
return {"ticks": 0}
return {
"ticks": _count,
"min_us": _min,
"mean_us": roundi(float(_total) / _count),
"p50_us": percentile(0.50),
"p95_us": percentile(0.95),
"p99_us": percentile(0.99),
"max_us": _max,
"budget_us": budget_usec,
"over_budget": over(budget_usec),
}
## How many ticks took longer than `budget_usec`.
func over(budget_usec: int) -> int:
var count: int = 0
if budget_usec < HISTOGRAM_USEC:
for usec: int in range(budget_usec + 1, HISTOGRAM_USEC):
count += _counts[usec]
for usec: int in _slow:
if usec > budget_usec:
count += 1
return count + _slow_dropped
func _sort_slow() -> void:
if not _slow_sorted:
_slow.sort()
_slow_sorted = true
+1
View File
@@ -0,0 +1 @@
uid://f8axsl0jjws6