H4: Long-run stability (make stability), and two memory leaks fixed
make stability (tools/stability.sh, StabilityReport) runs one dedicated server for 50 bot matches back to back, unthrottled, sampling its resident memory, and fails on a crash, hang, engine or script error, a match with no winner, or memory that grows after the first five matches. The first run had no crash, hang or error, but showed two leaks: - ServerLoop kept every tick's duration in an array (8 bytes a tick, about 21 MB a day at 30 Hz) and copied and sorted all of it for every progress line. TickTimes keeps exact statistics in fixed memory: a one-microsecond histogram up to 65 ms, plus the rare slower ticks. - Resident memory grew about 28 MB a match (905 MB after 42 matches): glibc gives each thread its own malloc arena, and every next island is built on a worker thread. The server launchers (packaging/server, packaging/dist, so also the .deb, Docker image and systemd unit) and tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625). After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no errors; static memory +1.6 KiB and resident +145 KiB a match, flat at 204 MB from match 26 on. No match crashed, so there is no crash seed to turn into a regression test. Tests: TickTimes against a sorted list, its fixed memory, the launchers' setting, and StabilityReport. make check, dist-check and server-package-check pass. Closes #57 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -63,4 +63,12 @@ if [[ "${TIDEWALL_GODOT_DISPLAY:-0}" != "1" ]]; then
|
||||
unset DISPLAY WAYLAND_DISPLAY
|
||||
fi
|
||||
|
||||
# A dedicated server gets one malloc arena, as the packaged launchers give it: see
|
||||
# packaging/server/tidewall-server (H4).
|
||||
for arg in "$@"; do
|
||||
if [[ "$arg" == "--server" ]]; then
|
||||
export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}"
|
||||
fi
|
||||
done
|
||||
|
||||
exec "$binary" "${args[@]}" "$@"
|
||||
|
||||
Executable
+59
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
# H4: long-run stability. One dedicated server process plays STABILITY_MATCHES (default
|
||||
# 50) whole bot matches back to back, unthrottled (accelerated time: the server never
|
||||
# waits for the clock), from seed STABILITY_SEED (default 4242), under nice. Each match
|
||||
# gets its own seed from the server's. The process's resident memory is sampled every
|
||||
# few seconds. tools/stability_report.gd then checks the log and samples: see
|
||||
# test/server/stability_report.gd for what passes. About 25 minutes on this host.
|
||||
#
|
||||
# tools/stability.sh
|
||||
# STABILITY_MATCHES=5 tools/stability.sh a quick look (too few to pass)
|
||||
#
|
||||
# If the host's available memory falls under 2 GB, the run is stopped and fails.
|
||||
set -uo pipefail
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$repo"
|
||||
matches="${STABILITY_MATCHES:-50}"
|
||||
seed="${STABILITY_SEED:-4242}"
|
||||
out="${STABILITY_DIR:-$repo/build/stability}"
|
||||
# A match is about 25,000 ticks; unthrottled that is well under a minute here. A hang is
|
||||
# anything over this many seconds of wall time per match.
|
||||
per_match_limit="${STABILITY_MATCH_SECONDS:-240}"
|
||||
limit=$(( matches * per_match_limit + 120 ))
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
|
||||
nice -n 15 tools/godot.sh -- --server --seed="$seed" --bots=24 --matches="$matches" --unthrottled \
|
||||
--log-every=3000 >"$out/server.log" 2>&1 &
|
||||
pid=$!
|
||||
started=$(date +%s)
|
||||
stopped=""
|
||||
: >"$out/rss.txt"
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
# The engine runs as a child of tools/godot.sh; sample the biggest process in the group.
|
||||
rss=0
|
||||
for p in "$pid" $(pgrep -P "$pid" 2>/dev/null); do
|
||||
r=$(awk '/^VmRSS:/ { print $2 }' "/proc/$p/status" 2>/dev/null || echo 0)
|
||||
(( ${r:-0} > rss )) && rss=${r:-0}
|
||||
done
|
||||
echo "$(( $(date +%s) - started )) $rss" >>"$out/rss.txt"
|
||||
avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo)
|
||||
if (( avail < 2048 )); then
|
||||
stopped="host memory low (${avail} MB available)"
|
||||
elif (( $(date +%s) - started > limit )); then
|
||||
stopped="over the ${limit} s limit: a hang"
|
||||
fi
|
||||
if [[ -n "$stopped" ]]; then
|
||||
pkill -TERM -P "$pid" 2>/dev/null; kill -TERM "$pid" 2>/dev/null
|
||||
sleep 3
|
||||
pkill -KILL -P "$pid" 2>/dev/null; kill -KILL "$pid" 2>/dev/null
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
wait "$pid" 2>/dev/null
|
||||
code=$?
|
||||
echo "$code" >"$out/exit_code.txt"
|
||||
[[ -n "$stopped" ]] && echo "$stopped" >"$out/stopped.txt"
|
||||
tools/godot.sh --script res://tools/stability_report.gd -- --dir="$out" --matches="$matches" \
|
||||
--match-seconds="$per_match_limit"
|
||||
@@ -0,0 +1,35 @@
|
||||
## Reads a tools/stability.sh run (server.log, rss.txt, exit_code.txt, stopped.txt) and
|
||||
## judges it with StabilityReport. Writes report.txt beside them. Exits 1 on a failure.
|
||||
##
|
||||
## tools/godot.sh --script res://tools/stability_report.gd -- --dir=PATH --matches=50
|
||||
extends SceneTree
|
||||
|
||||
|
||||
func _init() -> void:
|
||||
var dir_path: String = ""
|
||||
var matches: int = 50
|
||||
var match_seconds: float = 240.0
|
||||
for arg: String in OS.get_cmdline_user_args():
|
||||
if arg.begins_with("--dir="):
|
||||
dir_path = arg.trim_prefix("--dir=")
|
||||
elif arg.begins_with("--matches="):
|
||||
matches = arg.trim_prefix("--matches=").to_int()
|
||||
elif arg.begins_with("--match-seconds="):
|
||||
match_seconds = arg.trim_prefix("--match-seconds=").to_float()
|
||||
if dir_path == "":
|
||||
printerr("stability: give --dir=PATH")
|
||||
quit(2)
|
||||
return
|
||||
var log_lines: PackedStringArray = FileAccess.get_file_as_string(dir_path.path_join("server.log")).split("\n", false)
|
||||
var rss: PackedStringArray = FileAccess.get_file_as_string(dir_path.path_join("rss.txt")).split("\n", false)
|
||||
var code: int = FileAccess.get_file_as_string(dir_path.path_join("exit_code.txt")).strip_edges().to_int()
|
||||
var stopped: String = ""
|
||||
if FileAccess.file_exists(dir_path.path_join("stopped.txt")):
|
||||
stopped = FileAccess.get_file_as_string(dir_path.path_join("stopped.txt")).strip_edges()
|
||||
var a: Dictionary = StabilityReport.analyse(log_lines, rss, code, stopped, matches, match_seconds)
|
||||
var text: String = StabilityReport.format(a)
|
||||
var file: FileAccess = FileAccess.open(dir_path.path_join("report.txt"), FileAccess.WRITE)
|
||||
file.store_string(text)
|
||||
file.close()
|
||||
print(text)
|
||||
quit(0 if a["problems"].is_empty() else 1)
|
||||
@@ -0,0 +1 @@
|
||||
uid://cganord5usyws
|
||||
Reference in New Issue
Block a user