H4: Long-run stability (make stability), and two memory leaks fixed
make stability (tools/stability.sh, StabilityReport) runs one dedicated server for 50 bot matches back to back, unthrottled, sampling its resident memory, and fails on a crash, hang, engine or script error, a match with no winner, or memory that grows after the first five matches. The first run had no crash, hang or error, but showed two leaks: - ServerLoop kept every tick's duration in an array (8 bytes a tick, about 21 MB a day at 30 Hz) and copied and sorted all of it for every progress line. TickTimes keeps exact statistics in fixed memory: a one-microsecond histogram up to 65 ms, plus the rare slower ticks. - Resident memory grew about 28 MB a match (905 MB after 42 matches): glibc gives each thread its own malloc arena, and every next island is built on a worker thread. The server launchers (packaging/server, packaging/dist, so also the .deb, Docker image and systemd unit) and tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625). After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no errors; static memory +1.6 KiB and resident +145 KiB a match, flat at 204 MB from match 26 on. No match crashed, so there is no crash seed to turn into a regression test. Tests: TickTimes against a sorted list, its fixed memory, the launchers' setting, and StabilityReport. make check, dist-check and server-package-check pass. Closes #57 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,138 @@
|
||||
## Judges a long-run stability run (H4, tools/stability.sh): a dedicated server playing
|
||||
## matches back to back. Passes when every match ended with a winner inside the time
|
||||
## limit, the server exited cleanly with no engine or script error, and memory shows no
|
||||
## growth trend once the first matches have warmed the caches.
|
||||
class_name StabilityReport
|
||||
extends RefCounted
|
||||
|
||||
## Matches left out of the memory trend while pools and caches fill.
|
||||
const WARMUP_MATCHES: int = 5
|
||||
## The most the server's own (static) memory may grow per match, by least squares over
|
||||
## the matches after the warmup.
|
||||
const MAX_STATIC_GROWTH_PER_MATCH: float = 256.0 * 1024.0
|
||||
## The most the process's resident memory may grow per match, the same way. Looser,
|
||||
## since the allocator returns memory to the system when it likes.
|
||||
const MAX_RSS_GROWTH_PER_MATCH: float = 1024.0 * 1024.0
|
||||
|
||||
|
||||
## Least-squares slope of `values` against their index.
|
||||
static func slope(values: Array[float]) -> float:
|
||||
var n: int = values.size()
|
||||
if n < 2:
|
||||
return 0.0
|
||||
var mean_x: float = (n - 1) / 2.0
|
||||
var mean_y: float = 0.0
|
||||
for value: float in values:
|
||||
mean_y += value
|
||||
mean_y /= n
|
||||
var top: float = 0.0
|
||||
var bottom: float = 0.0
|
||||
for index: int in n:
|
||||
top += (index - mean_x) * (values[index] - mean_y)
|
||||
bottom += (index - mean_x) * (index - mean_x)
|
||||
return top / bottom
|
||||
|
||||
|
||||
## `log_lines`: the server's stdout and stderr. `rss_lines`: "seconds kilobytes" samples.
|
||||
## Returns {"matches": [{"seed", "index", "ticks", "seconds", "winners", "timed_out",
|
||||
## "static_mb", "rss_mb"}], "ended", "errors": [String], "exit_code", "stopped",
|
||||
## "static_slope", "rss_slope", "peak_rss_mb", "wall_seconds", "problems": [String]}.
|
||||
static func analyse(log_lines: PackedStringArray, rss_lines: PackedStringArray, exit_code: int, stopped: String, expected: int, match_seconds: float) -> Dictionary:
|
||||
var errors: Array[String] = []
|
||||
# A server without --listen starts its first match before it logs anything.
|
||||
var started: Dictionary = {"tick": 0, "t_ms": 0, "index": 0}
|
||||
var matches: Array[Dictionary] = []
|
||||
var stop: Dictionary = {}
|
||||
for line: String in log_lines:
|
||||
var parsed: Variant = JSON.parse_string(line) if line.begins_with("{") else null
|
||||
if not parsed is Dictionary:
|
||||
if line.contains("ERROR") or line.contains("SCRIPT ERROR") or line.contains("WARNING: ObjectDB instances leaked"):
|
||||
errors.append(line.strip_edges())
|
||||
continue
|
||||
var record: Dictionary = parsed
|
||||
var event: String = String(record.get("event", ""))
|
||||
if event == "match" and record.get("type", "") == "match_started":
|
||||
started = record
|
||||
elif event == "match" and record.get("type", "") == "match_ended" and not started.is_empty():
|
||||
matches.append({
|
||||
"seed": int(record["match"]), "index": int(started.get("index", matches.size())),
|
||||
"ticks": int(record["tick"]) - int(started["tick"]),
|
||||
"seconds": (float(record["t_ms"]) - float(started["t_ms"])) / 1000.0,
|
||||
"winners": record.get("winners", []), "timed_out": bool(record.get("timed_out", false)),
|
||||
"static_mb": float(record["mem_bytes"]) / 1048576.0, "t_s": float(record["t_ms"]) / 1000.0,
|
||||
})
|
||||
started = {}
|
||||
elif event == "stop":
|
||||
stop = record
|
||||
# Resident memory at each match's end: the last sample before it.
|
||||
var samples: Array[Vector2] = []
|
||||
for line: String in rss_lines:
|
||||
var parts: PackedStringArray = line.split(" ", false)
|
||||
if parts.size() == 2 and parts[1].to_int() > 0:
|
||||
samples.append(Vector2(parts[0].to_float(), parts[1].to_float() / 1024.0))
|
||||
var peak: float = 0.0
|
||||
for sample: Vector2 in samples:
|
||||
peak = maxf(peak, sample.y)
|
||||
for row: Dictionary in matches:
|
||||
row["rss_mb"] = 0.0
|
||||
for sample: Vector2 in samples:
|
||||
if sample.x <= row["t_s"] + 1.0:
|
||||
row["rss_mb"] = sample.y
|
||||
var statics: Array[float] = []
|
||||
var rss: Array[float] = []
|
||||
for index: int in range(WARMUP_MATCHES, matches.size()):
|
||||
statics.append(float(matches[index]["static_mb"]) * 1048576.0)
|
||||
if matches[index]["rss_mb"] > 0.0:
|
||||
rss.append(float(matches[index]["rss_mb"]) * 1048576.0)
|
||||
var a: Dictionary = {
|
||||
"matches": matches, "ended": matches.size(), "errors": errors, "exit_code": exit_code, "stopped": stopped,
|
||||
"static_slope": slope(statics), "rss_slope": slope(rss), "peak_rss_mb": peak,
|
||||
"wall_seconds": samples[-1].x if not samples.is_empty() else 0.0,
|
||||
"server_errors": int(stop.get("errors", -1)), "problems": [],
|
||||
}
|
||||
var problems: Array = a["problems"]
|
||||
if stopped != "":
|
||||
problems.append("the run was stopped: %s" % stopped)
|
||||
if matches.size() != expected:
|
||||
problems.append("%d of %d matches ended" % [matches.size(), expected])
|
||||
if exit_code != 0:
|
||||
problems.append("the server exited %d" % exit_code)
|
||||
if stop.is_empty():
|
||||
problems.append("the server never logged its stop")
|
||||
elif int(stop.get("errors", 0)) != 0:
|
||||
problems.append("the server counted %d errors, first: %s" % [stop["errors"], stop.get("first_error", "")])
|
||||
if not errors.is_empty():
|
||||
problems.append("%d error lines, first: %s" % [errors.size(), errors[0]])
|
||||
for row: Dictionary in matches:
|
||||
if row["winners"].is_empty():
|
||||
problems.append("match %d (seed %d) ended with no winner" % [row["index"], row["seed"]])
|
||||
if row["seconds"] > match_seconds:
|
||||
problems.append("match %d (seed %d) took %.0f s, over the %.0f s hang limit" % [row["index"], row["seed"], row["seconds"], match_seconds])
|
||||
if matches.size() >= WARMUP_MATCHES + 10:
|
||||
if a["static_slope"] > MAX_STATIC_GROWTH_PER_MATCH:
|
||||
problems.append("static memory grows %.0f KiB a match (limit %.0f)" % [a["static_slope"] / 1024.0, MAX_STATIC_GROWTH_PER_MATCH / 1024.0])
|
||||
if a["rss_slope"] > MAX_RSS_GROWTH_PER_MATCH:
|
||||
problems.append("resident memory grows %.0f KiB a match (limit %.0f)" % [a["rss_slope"] / 1024.0, MAX_RSS_GROWTH_PER_MATCH / 1024.0])
|
||||
else:
|
||||
problems.append("only %d matches; the memory trend needs at least %d" % [matches.size(), WARMUP_MATCHES + 10])
|
||||
return a
|
||||
|
||||
|
||||
static func format(a: Dictionary) -> String:
|
||||
var lines: PackedStringArray = []
|
||||
lines.append("Tidewall long-run stability: %d matches back to back on one server, unthrottled" % a["ended"])
|
||||
lines.append(" wall time %.0f s, exit code %d, server errors %d, error lines %d" % [a["wall_seconds"], a["exit_code"], a["server_errors"], a["errors"].size()])
|
||||
lines.append(" memory after the first %d matches: static %+.1f KiB a match, resident %+.1f KiB a match; peak resident %.0f MB" % [
|
||||
WARMUP_MATCHES, a["static_slope"] / 1024.0, a["rss_slope"] / 1024.0, a["peak_rss_mb"]])
|
||||
lines.append("")
|
||||
lines.append(" # seed ticks wall s winner static MB resident MB")
|
||||
for row: Dictionary in a["matches"]:
|
||||
lines.append(" %-3d %-20d %6d %7.1f %7s %10.1f %12.1f%s" % [row["index"], row["seed"], row["ticks"], row["seconds"],
|
||||
"yes" if not row["winners"].is_empty() else "NO", row["static_mb"], row["rss_mb"], " (tide's clock)" if row["timed_out"] else ""])
|
||||
lines.append("")
|
||||
if a["problems"].is_empty():
|
||||
lines.append("stability: ok")
|
||||
else:
|
||||
for problem: String in a["problems"]:
|
||||
lines.append("stability FAIL: %s" % problem)
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1 @@
|
||||
uid://b8e85xf8plubp
|
||||
@@ -0,0 +1,64 @@
|
||||
extends TestCase
|
||||
|
||||
## The long-run stability report (H4): what counts as a clean run of back-to-back
|
||||
## matches, and that a memory trend, a hang, an error or a missing match fails it.
|
||||
|
||||
|
||||
func _log(matches: int, static_growth: float, seconds: float = 20.0, winners: bool = true) -> PackedStringArray:
|
||||
var lines: PackedStringArray = ["Godot Engine v4.7.2.stable.official - https://godotengine.org"]
|
||||
var tick: int = 0
|
||||
var t_ms: float = 0.0
|
||||
for index: int in matches:
|
||||
if index > 0:
|
||||
lines.append(JSON.stringify({"event": "match", "type": "match_started", "match": 1000 + index, "index": index, "tick": tick, "t_ms": t_ms, "mem_bytes": 70000000}))
|
||||
tick += 25000
|
||||
t_ms += seconds * 1000.0
|
||||
lines.append(JSON.stringify({"event": "match", "type": "match_ended", "match": 1000 + index, "winners": [-3] if winners else [], "timed_out": false,
|
||||
"tick": tick, "t_ms": t_ms, "mem_bytes": 70000000 + roundi(static_growth * index)}))
|
||||
lines.append(JSON.stringify({"event": "stop", "exit_code": 0, "errors": 0, "tick": tick, "t_ms": t_ms}))
|
||||
return lines
|
||||
|
||||
|
||||
func _rss(matches: int, growth_kb: float, seconds: float = 20.0) -> PackedStringArray:
|
||||
var lines: PackedStringArray = []
|
||||
for index: int in matches * 4 + 1:
|
||||
var at: float = index * seconds / 4.0
|
||||
lines.append("%d %d" % [roundi(at), 200000 + roundi(growth_kb * at / seconds)])
|
||||
return lines
|
||||
|
||||
|
||||
func test_slope_is_least_squares() -> void:
|
||||
assert_eq(StabilityReport.slope([1.0, 3.0, 5.0] as Array[float]), 2.0)
|
||||
assert_eq(StabilityReport.slope([4.0, 4.0, 4.0, 4.0] as Array[float]), 0.0)
|
||||
assert_eq(StabilityReport.slope([7.0] as Array[float]), 0.0)
|
||||
|
||||
|
||||
func test_a_clean_flat_run_passes() -> void:
|
||||
var a: Dictionary = StabilityReport.analyse(_log(50, 1000.0), _rss(50, 10.0), 0, "", 50, 240.0)
|
||||
assert_eq(a["problems"], [], "%s" % [a["problems"]])
|
||||
assert_eq(a["ended"], 50)
|
||||
assert_eq(a["matches"][0]["ticks"], 25000, "the first match counts from the start")
|
||||
assert_near(a["static_slope"], 1000.0, 1.0)
|
||||
assert_true(a["rss_slope"] > 0.0 and a["rss_slope"] < 20.0 * 1024.0)
|
||||
assert_true(StabilityReport.format(a).contains("stability: ok"))
|
||||
|
||||
|
||||
func test_growth_hangs_errors_and_missing_matches_fail() -> void:
|
||||
var leak: Dictionary = StabilityReport.analyse(_log(50, 512.0 * 1024.0), _rss(50, 0.0), 0, "", 50, 240.0)
|
||||
assert_eq(leak["problems"].size(), 1, "%s" % [leak["problems"]])
|
||||
assert_true(String(leak["problems"][0]).begins_with("static memory grows 512 KiB"))
|
||||
var rss: Dictionary = StabilityReport.analyse(_log(50, 0.0), _rss(50, 4096.0), 0, "", 50, 240.0)
|
||||
assert_true(String(rss["problems"][0]).begins_with("resident memory grows"), "%s" % [rss["problems"]])
|
||||
var slow: Dictionary = StabilityReport.analyse(_log(20, 0.0, 300.0), _rss(20, 0.0, 300.0), 0, "", 20, 240.0)
|
||||
assert_eq(slow["problems"].size(), 20, "every match over the hang limit")
|
||||
var lines: PackedStringArray = _log(20, 0.0)
|
||||
lines.insert(3, "SCRIPT ERROR: Invalid call. Nonexistent function 'x' in base 'Nil'.")
|
||||
var broken: Dictionary = StabilityReport.analyse(lines, _rss(20, 0.0), 1, "", 20, 240.0)
|
||||
var text: String = StabilityReport.format(broken)
|
||||
assert_true(text.contains("the server exited 1"))
|
||||
assert_true(text.contains("1 error lines, first: SCRIPT ERROR"))
|
||||
var short: Dictionary = StabilityReport.analyse(_log(30, 0.0, 20.0, false), _rss(30, 0.0), 0, "over the 7320 s limit: a hang", 50, 240.0)
|
||||
var short_text: String = StabilityReport.format(short)
|
||||
assert_true(short_text.contains("the run was stopped: over the 7320 s limit"))
|
||||
assert_true(short_text.contains("30 of 50 matches ended"))
|
||||
assert_true(short_text.contains("ended with no winner"))
|
||||
@@ -0,0 +1 @@
|
||||
uid://c26lqrxplefwk
|
||||
@@ -0,0 +1,40 @@
|
||||
extends TestCase
|
||||
|
||||
## H4: tick timing in fixed memory, and the server's launchers keeping the allocator to
|
||||
## one arena so resident memory doesn't grow match after match.
|
||||
|
||||
|
||||
func test_statistics_match_a_sorted_list() -> void:
|
||||
var times: TickTimes = TickTimes.new()
|
||||
var values: PackedInt64Array = PackedInt64Array()
|
||||
var rng: RandomNumberGenerator = RandomNumberGenerator.new()
|
||||
rng.seed = 4
|
||||
for index: int in 5000:
|
||||
var usec: int = rng.randi_range(200, 3000) if index % 500 != 0 else rng.randi_range(70000, 2000000)
|
||||
times.add(usec)
|
||||
values.append(usec)
|
||||
values.sort()
|
||||
for fraction: float in [0.0, 0.01, 0.5, 0.95, 0.99, 0.999, 1.0]:
|
||||
assert_eq(times.percentile(fraction), ServerLoop._percentile(values, fraction), "percentile %.3f" % fraction)
|
||||
var stats: Dictionary = times.stats(ServerLoop.TICK_BUDGET_USEC)
|
||||
assert_eq(stats["ticks"], 5000)
|
||||
assert_eq(stats["min_us"], values[0])
|
||||
assert_eq(stats["max_us"], values[-1])
|
||||
assert_eq(stats["over_budget"], 10, "the ten slow ticks")
|
||||
assert_eq(TickTimes.new().stats(1000), {"ticks": 0})
|
||||
|
||||
|
||||
func test_memory_stays_fixed_past_the_slow_tick_cap() -> void:
|
||||
var times: TickTimes = TickTimes.new()
|
||||
for index: int in TickTimes.MAX_SLOW + 100:
|
||||
times.add(TickTimes.HISTOGRAM_USEC + index)
|
||||
assert_eq(times._slow.size(), TickTimes.MAX_SLOW)
|
||||
assert_eq(times.size(), TickTimes.MAX_SLOW + 100)
|
||||
assert_eq(times.percentile(1.0), TickTimes.HISTOGRAM_USEC + TickTimes.MAX_SLOW + 99, "the slowest are kept")
|
||||
assert_eq(times.over(12000), TickTimes.MAX_SLOW + 100)
|
||||
|
||||
|
||||
func test_server_launchers_use_one_malloc_arena() -> void:
|
||||
for path: String in ["res://packaging/server/tidewall-server", "res://packaging/dist/tidewall-server", "res://tools/godot.sh"]:
|
||||
var text: String = FileAccess.get_file_as_string(path)
|
||||
assert_true(text.contains('export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}"'), "%s sets MALLOC_ARENA_MAX" % path)
|
||||
@@ -0,0 +1 @@
|
||||
uid://2ib5r4fuuxpe
|
||||
Reference in New Issue
Block a user