Files
tidewall/test/server/test_stability_report.gd
claudeandClaude Opus 5.5 f87d5eae0d H4: Long-run stability (make stability), and two memory leaks fixed
make stability (tools/stability.sh, StabilityReport) runs one dedicated
server for 50 bot matches back to back, unthrottled, sampling its resident
memory, and fails on a crash, hang, engine or script error, a match with
no winner, or memory that grows after the first five matches.

The first run had no crash, hang or error, but showed two leaks:
- ServerLoop kept every tick's duration in an array (8 bytes a tick,
  about 21 MB a day at 30 Hz) and copied and sorted all of it for every
  progress line. TickTimes keeps exact statistics in fixed memory: a
  one-microsecond histogram up to 65 ms, plus the rare slower ticks.
- Resident memory grew about 28 MB a match (905 MB after 42 matches):
  glibc gives each thread its own malloc arena, and every next island is
  built on a worker thread. The server launchers (packaging/server,
  packaging/dist, so also the .deb, Docker image and systemd unit) and
  tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already
  set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625).

After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no
errors; static memory +1.6 KiB and resident +145 KiB a match, flat at
204 MB from match 26 on. No match crashed, so there is no crash seed to
turn into a regression test.

Tests: TickTimes against a sorted list, its fixed memory, the launchers'
setting, and StabilityReport. make check, dist-check and
server-package-check pass.

Closes #57

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-14 07:15:23 -05:00

65 lines
3.5 KiB
GDScript

extends TestCase
## The long-run stability report (H4): what counts as a clean run of back-to-back
## matches, and that a memory trend, a hang, an error or a missing match fails it.
func _log(matches: int, static_growth: float, seconds: float = 20.0, winners: bool = true) -> PackedStringArray:
var lines: PackedStringArray = ["Godot Engine v4.7.2.stable.official - https://godotengine.org"]
var tick: int = 0
var t_ms: float = 0.0
for index: int in matches:
if index > 0:
lines.append(JSON.stringify({"event": "match", "type": "match_started", "match": 1000 + index, "index": index, "tick": tick, "t_ms": t_ms, "mem_bytes": 70000000}))
tick += 25000
t_ms += seconds * 1000.0
lines.append(JSON.stringify({"event": "match", "type": "match_ended", "match": 1000 + index, "winners": [-3] if winners else [], "timed_out": false,
"tick": tick, "t_ms": t_ms, "mem_bytes": 70000000 + roundi(static_growth * index)}))
lines.append(JSON.stringify({"event": "stop", "exit_code": 0, "errors": 0, "tick": tick, "t_ms": t_ms}))
return lines
func _rss(matches: int, growth_kb: float, seconds: float = 20.0) -> PackedStringArray:
var lines: PackedStringArray = []
for index: int in matches * 4 + 1:
var at: float = index * seconds / 4.0
lines.append("%d %d" % [roundi(at), 200000 + roundi(growth_kb * at / seconds)])
return lines
func test_slope_is_least_squares() -> void:
assert_eq(StabilityReport.slope([1.0, 3.0, 5.0] as Array[float]), 2.0)
assert_eq(StabilityReport.slope([4.0, 4.0, 4.0, 4.0] as Array[float]), 0.0)
assert_eq(StabilityReport.slope([7.0] as Array[float]), 0.0)
func test_a_clean_flat_run_passes() -> void:
var a: Dictionary = StabilityReport.analyse(_log(50, 1000.0), _rss(50, 10.0), 0, "", 50, 240.0)
assert_eq(a["problems"], [], "%s" % [a["problems"]])
assert_eq(a["ended"], 50)
assert_eq(a["matches"][0]["ticks"], 25000, "the first match counts from the start")
assert_near(a["static_slope"], 1000.0, 1.0)
assert_true(a["rss_slope"] > 0.0 and a["rss_slope"] < 20.0 * 1024.0)
assert_true(StabilityReport.format(a).contains("stability: ok"))
func test_growth_hangs_errors_and_missing_matches_fail() -> void:
var leak: Dictionary = StabilityReport.analyse(_log(50, 512.0 * 1024.0), _rss(50, 0.0), 0, "", 50, 240.0)
assert_eq(leak["problems"].size(), 1, "%s" % [leak["problems"]])
assert_true(String(leak["problems"][0]).begins_with("static memory grows 512 KiB"))
var rss: Dictionary = StabilityReport.analyse(_log(50, 0.0), _rss(50, 4096.0), 0, "", 50, 240.0)
assert_true(String(rss["problems"][0]).begins_with("resident memory grows"), "%s" % [rss["problems"]])
var slow: Dictionary = StabilityReport.analyse(_log(20, 0.0, 300.0), _rss(20, 0.0, 300.0), 0, "", 20, 240.0)
assert_eq(slow["problems"].size(), 20, "every match over the hang limit")
var lines: PackedStringArray = _log(20, 0.0)
lines.insert(3, "SCRIPT ERROR: Invalid call. Nonexistent function 'x' in base 'Nil'.")
var broken: Dictionary = StabilityReport.analyse(lines, _rss(20, 0.0), 1, "", 20, 240.0)
var text: String = StabilityReport.format(broken)
assert_true(text.contains("the server exited 1"))
assert_true(text.contains("1 error lines, first: SCRIPT ERROR"))
var short: Dictionary = StabilityReport.analyse(_log(30, 0.0, 20.0, false), _rss(30, 0.0), 0, "over the 7320 s limit: a hang", 50, 240.0)
var short_text: String = StabilityReport.format(short)
assert_true(short_text.contains("the run was stopped: over the 7320 s limit"))
assert_true(short_text.contains("30 of 50 matches ended"))
assert_true(short_text.contains("ended with no winner"))