From f87d5eae0dfba8237e879bc02a839dc2cb7a5da4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 14 Nov 2026 07:15:23 -0500 Subject: [PATCH] H4: Long-run stability (make stability), and two memory leaks fixed make stability (tools/stability.sh, StabilityReport) runs one dedicated server for 50 bot matches back to back, unthrottled, sampling its resident memory, and fails on a crash, hang, engine or script error, a match with no winner, or memory that grows after the first five matches. The first run had no crash, hang or error, but showed two leaks: - ServerLoop kept every tick's duration in an array (8 bytes a tick, about 21 MB a day at 30 Hz) and copied and sorted all of it for every progress line. TickTimes keeps exact statistics in fixed memory: a one-microsecond histogram up to 65 ms, plus the rare slower ticks. - Resident memory grew about 28 MB a match (905 MB after 42 matches): glibc gives each thread its own malloc arena, and every next island is built on a worker thread. The server launchers (packaging/server, packaging/dist, so also the .deb, Docker image and systemd unit) and tools/godot.sh for --server now set MALLOC_ARENA_MAX=1 unless already set. Tick times were unchanged (p50 501 vs 507 us, p99 1635 vs 1625). After: 50 matches (seed 4242) in 756 s, all with a winner, exit 0, no errors; static memory +1.6 KiB and resident +145 KiB a match, flat at 204 MB from match 26 on. No match crashed, so there is no crash seed to turn into a regression test. Tests: TickTimes against a sorted list, its fixed memory, the launchers' setting, and StabilityReport. make check, dist-check and server-package-check pass. Closes #57 Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 1 + Makefile | 6 +- packaging/README.md | 8 ++ packaging/dist/tidewall-server | 5 + packaging/server/tidewall-server | 5 + src/server/headless_server.gd | 14 +-- src/server/server_loop.gd | 29 +---- src/server/tick_times.gd | 103 +++++++++++++++++ src/server/tick_times.gd.uid | 1 + test/server/stability_report.gd | 138 +++++++++++++++++++++++ test/server/stability_report.gd.uid | 1 + test/server/test_stability_report.gd | 64 +++++++++++ test/server/test_stability_report.gd.uid | 1 + test/server/test_tick_times.gd | 40 +++++++ test/server/test_tick_times.gd.uid | 1 + tools/godot.sh | 8 ++ tools/stability.sh | 59 ++++++++++ tools/stability_report.gd | 35 ++++++ tools/stability_report.gd.uid | 1 + 19 files changed, 486 insertions(+), 34 deletions(-) create mode 100644 src/server/tick_times.gd create mode 100644 src/server/tick_times.gd.uid create mode 100644 test/server/stability_report.gd create mode 100644 test/server/stability_report.gd.uid create mode 100644 test/server/test_stability_report.gd create mode 100644 test/server/test_stability_report.gd.uid create mode 100644 test/server/test_tick_times.gd create mode 100644 test/server/test_tick_times.gd.uid create mode 100755 tools/stability.sh create mode 100644 tools/stability_report.gd create mode 100644 tools/stability_report.gd.uid diff --git a/CLAUDE.md b/CLAUDE.md index 361bf11..75e8f2b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -48,6 +48,7 @@ continuity. | `make build-captures` | Renders the fort and the wall test card: a row per material, a column per state (half built, intact, worn, battered), and a battered ramp (`captures/build_*.png`), and the barge and sea chests (`captures/vessels.png`), about 2 min. Needs Docker. Look at them. | | `tools/capture.sh canopy captures/canopy.png` | H11: the shoulder camera on a player against a pine, with canopies in the way hidden (`canopy_before` puts them back). | | `make style-captures` | Renders a hamlet and a dock with and without the woodcut print pass (`captures/style_*.png`). Needs Docker. | +| `make stability` | H4: one dedicated server plays 50 bot matches back to back, unthrottled, under `nice` (about 13 min), sampling its resident memory. Fails on a crash, hang, engine or script error, a match without a winner, or memory growing more than 256 KiB (static) or 1 MiB (resident) a match after the first five. `STABILITY_MATCHES`, `STABILITY_SEED`, `STABILITY_DIR` scale it. Report in `build/stability/report.txt`. | | `make soak` | The 24-client network soak (N8): a real server and 24 headless `SoakPlayer` clients in three processes, each behind a `UdpRelay` adding 60 ms, +/-20 ms jitter and 2% loss each way, for two whole matches in real time (about 30 min, under `nice`). Fails on a world-hash mismatch, errors, unreadable packets, a client that didn't play every match, memory growth between matches, or a p99 tick over budget. `SOAK_MATCHES`, `SOAK_CLIENTS`, `SOAK_PROCS`, `SOAK_LATENCY`, `SOAK_JITTER`, `SOAK_LOSS` scale it. Results in `build/soak/report.txt`. | | `make bench` | H1: the server's p50/p99/max tick with 24 bots on three seeds, and the client's draw calls, triangles, objects and memory at six moments of a match (about 3 min, needs Docker). Fails on anything over the budgets in docs/ARCHITECTURE.md. Report in `build/bench/report.txt`. `BENCH_SEEDS`, `BENCH_TICKS`, `BENCH_CLIENT=0` scale it. | | `make dist` | R1: exports the game with the official Godot templates (checked against `tools/engine.sha512`, installed into `.godot-home/`) to `build/dist/tidewall--linux-x86_64.tar.gz` with a `.sha256`. One binary and one pack: `tidewall.x86_64` is the game, `tidewall-server` runs it headless as the server. The version is `application/config/version` in project.godot; the commit goes in as `build_commit.txt` (`BuildInfo`). | diff --git a/Makefile b/Makefile index 10c5015..d9f08df 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Tidewall build entry points. Everything runs headless with the repo-local engine. GODOT := tools/godot.sh -.PHONY: app-icon deb deb-check bench dist dist-check build-captures poi-captures art figure figure-captures sea-captures models item-captures audio audio-check check import parse test test-selfcheck smoke capture-test terrain-capture binaries verify-engine balance bot-matches match match-captures client-check client-captures hud-captures style-captures menu-captures chart-captures discovery-check fuzz soak soak-selfcheck server-package-check +.PHONY: app-icon deb deb-check bench dist dist-check build-captures poi-captures art figure figure-captures sea-captures models item-captures audio audio-check check import parse test test-selfcheck smoke capture-test terrain-capture binaries verify-engine balance bot-matches match match-captures client-check client-captures hud-captures style-captures menu-captures chart-captures discovery-check fuzz stability soak soak-selfcheck server-package-check ## A8: regenerates every sound in assets/audio/ from tools/audio/ (about 6 s). audio: import @@ -172,6 +172,10 @@ terrain-capture: import ## The 24-client network soak: a real server and 24 headless clients behind a relay that ## adds latency, jitter and loss, for SOAK_MATCHES (default 2) whole matches in real ## time, about half an hour. Checks for desyncs, errors, leaks and the tick budget. +## H4: 50 bot matches back to back on one unthrottled server; report in build/stability/report.txt. +stability: import + @tools/stability.sh + soak: import @tools/soak.sh diff --git a/packaging/README.md b/packaging/README.md index b3f1dd4..fbd7894 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -60,6 +60,14 @@ Stopping: send SIGTERM to the wrapper (Docker, Compose and systemd all do). It c the server's stop file, the server disconnects everyone with a "shut down" reason, logs its statistics and exits 0. If it hasn't gone within 15 seconds the wrapper kills it. +## Memory + +The launchers set `MALLOC_ARENA_MAX=1` unless it is already set. The server builds each +next island on a worker thread, and with glibc's default of an arena per thread its +resident memory grew about 28 MB a match for as long as it ran. With one arena it stays +flat, about 200 MB with 24 bots, at no measured cost to tick time. `make stability` +checks this over 50 matches. + ## Ports | Port | Protocol | What | diff --git a/packaging/dist/tidewall-server b/packaging/dist/tidewall-server index 3e56db2..067d228 100755 --- a/packaging/dist/tidewall-server +++ b/packaging/dist/tidewall-server @@ -17,6 +17,11 @@ state="${TIDEWALL_STATE:-${XDG_STATE_HOME:-$HOME/.local/state}/tidewall-server}" grace="${TIDEWALL_STOP_SECONDS:-15}" stop_file="$state/stop" +# glibc gives each thread that allocates its own malloc arena and rarely hands the +# memory back. The server builds every next island on a worker thread, so with the +# default arenas its resident memory grew about 28 MB a match, forever (H4, #57). +# One arena keeps it flat at no measured cost to tick time. +export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" mkdir -p "$state/data" "$state/config" "$state/cache" rm -f "$stop_file" export XDG_DATA_HOME="$state/data" XDG_CONFIG_HOME="$state/config" XDG_CACHE_HOME="$state/cache" diff --git a/packaging/server/tidewall-server b/packaging/server/tidewall-server index 6285c01..ec5927e 100755 --- a/packaging/server/tidewall-server +++ b/packaging/server/tidewall-server @@ -22,6 +22,11 @@ state="${TIDEWALL_STATE:-${RUNTIME_DIRECTORY:-/tmp/tidewall-server}}" grace="${TIDEWALL_STOP_SECONDS:-15}" stop_file="$state/stop" +# glibc gives each thread that allocates its own malloc arena and rarely hands the +# memory back. The server builds every next island on a worker thread, so with the +# default arenas its resident memory grew about 28 MB a match, forever (H4, #57). +# One arena keeps it flat at no measured cost to tick time. +export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" mkdir -p "$state/data" "$state/config" "$state/cache" rm -f "$stop_file" # The engine writes its user data and caches here, so the install can stay read-only. diff --git a/src/server/headless_server.gd b/src/server/headless_server.gd index 97bd89d..e5294e4 100644 --- a/src/server/headless_server.gd +++ b/src/server/headless_server.gd @@ -34,7 +34,7 @@ var _finished: bool = false var _matches_ended: int = 0 ## When listening: microseconds each tick took including the network's share (after_tick: ## world messages and snapshots), which ServerLoop's own timing leaves out. -var _net_tick_usec: PackedInt64Array = PackedInt64Array() +var _net_tick_usec: TickTimes = TickTimes.new() func _init(server_config: ServerConfig) -> void: @@ -191,7 +191,7 @@ func _tick() -> void: loop.run_tick() if game != null: game.after_tick() - _net_tick_usec.append(Time.get_ticks_usec() - began) + _net_tick_usec.add(Time.get_ticks_usec() - began) if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS: _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)}) for event: Dictionary in loop.match_flow.events: @@ -223,12 +223,10 @@ func _finish(requested_code: int) -> void: var stats: Dictionary = loop.timing_stats() stats["state_hash"] = loop.sim.state_hash() stats["entities"] = loop.sim.entities.count() - if not _net_tick_usec.is_empty(): - var sorted: PackedInt64Array = _net_tick_usec.duplicate() - sorted.sort() - stats["with_net_p50_us"] = ServerLoop._percentile(sorted, 0.50) - stats["with_net_p99_us"] = ServerLoop._percentile(sorted, 0.99) - stats["with_net_max_us"] = sorted[sorted.size() - 1] + if _net_tick_usec.size() > 0: + stats["with_net_p50_us"] = _net_tick_usec.percentile(0.50) + stats["with_net_p99_us"] = _net_tick_usec.percentile(0.99) + stats["with_net_max_us"] = _net_tick_usec.stats(ServerLoop.TICK_BUDGET_USEC)["max_us"] stats["mem_bytes"] = OS.get_static_memory_usage() _log("stats", stats) if code == 0 and _errors.count() > 0: diff --git a/src/server/server_loop.gd b/src/server/server_loop.gd index 70a5feb..cfab29f 100644 --- a/src/server/server_loop.gd +++ b/src/server/server_loop.gd @@ -24,8 +24,8 @@ var nav: NavGrid = null var world: BotWorld = null ## Every bot's difficulty, or -1 to mix them: easy, normal, hard, in turn. var bot_difficulty: int = -1 -## Wall-clock duration of every tick run, in microseconds. -var tick_usec: PackedInt64Array = PackedInt64Array() +## Wall-clock duration of every tick run, in microseconds, in fixed memory. +var tick_times: TickTimes = TickTimes.new() ## Server ticks run, across every match. var ticks: int = 0 @@ -102,7 +102,7 @@ func run_tick() -> void: submit(match_flow.handle_of[bot.player_id], bot.think(sim, nav, world)) _advance() ticks += 1 - tick_usec.append(Time.get_ticks_usec() - started) + tick_times.add(Time.get_ticks_usec() - started) func _advance() -> void: @@ -150,28 +150,7 @@ func _track_players() -> void: ## Timing statistics over every tick run so far, in microseconds. func timing_stats() -> Dictionary: - var count: int = tick_usec.size() - if count == 0: - return {"ticks": 0} - var sorted: PackedInt64Array = tick_usec.duplicate() - sorted.sort() - var total: int = 0 - var over_budget: int = 0 - for value: int in sorted: - total += value - if value > TICK_BUDGET_USEC: - over_budget += 1 - return { - "ticks": count, - "min_us": sorted[0], - "mean_us": roundi(float(total) / count), - "p50_us": _percentile(sorted, 0.50), - "p95_us": _percentile(sorted, 0.95), - "p99_us": _percentile(sorted, 0.99), - "max_us": sorted[count - 1], - "budget_us": TICK_BUDGET_USEC, - "over_budget": over_budget, - } + return tick_times.stats(TICK_BUDGET_USEC) ## Nearest-rank percentile of an ascending array. diff --git a/src/server/tick_times.gd b/src/server/tick_times.gd new file mode 100644 index 0000000..d8a3727 --- /dev/null +++ b/src/server/tick_times.gd @@ -0,0 +1,103 @@ +## Tick durations in microseconds, kept in fixed memory however long the server runs +## (H4: a list of every tick grew by 8 bytes a tick, about 21 MB a day, and was copied +## and sorted for every progress line). A one-microsecond histogram up to HISTOGRAM_USEC +## makes every statistic exact below that; the few ticks above it (a lobby waiting on +## the next island, say) are kept as they are, up to MAX_SLOW of them. +class_name TickTimes +extends RefCounted + +const HISTOGRAM_USEC: int = 65536 +const MAX_SLOW: int = 4096 + +var _counts: PackedInt32Array = PackedInt32Array() +## Ticks at or over HISTOGRAM_USEC, ascending once sorted. +var _slow: PackedInt64Array = PackedInt64Array() +var _slow_sorted: bool = true +var _slow_dropped: int = 0 +var _count: int = 0 +var _total: int = 0 +var _min: int = 0 +var _max: int = 0 + + +func _init() -> void: + _counts.resize(HISTOGRAM_USEC) + + +func add(usec: int) -> void: + usec = maxi(usec, 0) + if _count == 0: + _min = usec + _max = usec + _min = mini(_min, usec) + _max = maxi(_max, usec) + _count += 1 + _total += usec + if usec < HISTOGRAM_USEC: + _counts[usec] += 1 + elif _slow.size() < MAX_SLOW: + _slow.append(usec) + _slow_sorted = false + else: + # Past the cap, keep the slowest: drop the fastest slow tick if this one is slower. + _sort_slow() + _slow_dropped += 1 + if usec > _slow[0]: + _slow[0] = usec + _slow_sorted = false + + +func size() -> int: + return _count + + +## Nearest-rank percentile, as ServerLoop._percentile gives over a sorted list. +func percentile(fraction: float) -> int: + if _count == 0: + return 0 + var rank: int = clampi(ceili(fraction * _count) - 1, 0, _count - 1) + var seen: int = 0 + for usec: int in HISTOGRAM_USEC: + seen += _counts[usec] + if seen > rank: + return usec + _sort_slow() + # Ranks among the slow ticks; any dropped past the cap were the fastest of them. + var index: int = rank - seen - _slow_dropped + return _slow[clampi(index, 0, _slow.size() - 1)] if not _slow.is_empty() else _max + + +## {"ticks", "min_us", "mean_us", "p50_us", "p95_us", "p99_us", "max_us", "budget_us", +## "over_budget"}, or {"ticks": 0} before the first tick. +func stats(budget_usec: int) -> Dictionary: + if _count == 0: + return {"ticks": 0} + return { + "ticks": _count, + "min_us": _min, + "mean_us": roundi(float(_total) / _count), + "p50_us": percentile(0.50), + "p95_us": percentile(0.95), + "p99_us": percentile(0.99), + "max_us": _max, + "budget_us": budget_usec, + "over_budget": over(budget_usec), + } + + +## How many ticks took longer than `budget_usec`. +func over(budget_usec: int) -> int: + var count: int = 0 + if budget_usec < HISTOGRAM_USEC: + for usec: int in range(budget_usec + 1, HISTOGRAM_USEC): + count += _counts[usec] + for usec: int in _slow: + if usec > budget_usec: + count += 1 + return count + _slow_dropped + + +func _sort_slow() -> void: + if not _slow_sorted: + _slow.sort() + _slow_sorted = true diff --git a/src/server/tick_times.gd.uid b/src/server/tick_times.gd.uid new file mode 100644 index 0000000..fd2655b --- /dev/null +++ b/src/server/tick_times.gd.uid @@ -0,0 +1 @@ +uid://f8axsl0jjws6 diff --git a/test/server/stability_report.gd b/test/server/stability_report.gd new file mode 100644 index 0000000..2437873 --- /dev/null +++ b/test/server/stability_report.gd @@ -0,0 +1,138 @@ +## Judges a long-run stability run (H4, tools/stability.sh): a dedicated server playing +## matches back to back. Passes when every match ended with a winner inside the time +## limit, the server exited cleanly with no engine or script error, and memory shows no +## growth trend once the first matches have warmed the caches. +class_name StabilityReport +extends RefCounted + +## Matches left out of the memory trend while pools and caches fill. +const WARMUP_MATCHES: int = 5 +## The most the server's own (static) memory may grow per match, by least squares over +## the matches after the warmup. +const MAX_STATIC_GROWTH_PER_MATCH: float = 256.0 * 1024.0 +## The most the process's resident memory may grow per match, the same way. Looser, +## since the allocator returns memory to the system when it likes. +const MAX_RSS_GROWTH_PER_MATCH: float = 1024.0 * 1024.0 + + +## Least-squares slope of `values` against their index. +static func slope(values: Array[float]) -> float: + var n: int = values.size() + if n < 2: + return 0.0 + var mean_x: float = (n - 1) / 2.0 + var mean_y: float = 0.0 + for value: float in values: + mean_y += value + mean_y /= n + var top: float = 0.0 + var bottom: float = 0.0 + for index: int in n: + top += (index - mean_x) * (values[index] - mean_y) + bottom += (index - mean_x) * (index - mean_x) + return top / bottom + + +## `log_lines`: the server's stdout and stderr. `rss_lines`: "seconds kilobytes" samples. +## Returns {"matches": [{"seed", "index", "ticks", "seconds", "winners", "timed_out", +## "static_mb", "rss_mb"}], "ended", "errors": [String], "exit_code", "stopped", +## "static_slope", "rss_slope", "peak_rss_mb", "wall_seconds", "problems": [String]}. +static func analyse(log_lines: PackedStringArray, rss_lines: PackedStringArray, exit_code: int, stopped: String, expected: int, match_seconds: float) -> Dictionary: + var errors: Array[String] = [] + # A server without --listen starts its first match before it logs anything. + var started: Dictionary = {"tick": 0, "t_ms": 0, "index": 0} + var matches: Array[Dictionary] = [] + var stop: Dictionary = {} + for line: String in log_lines: + var parsed: Variant = JSON.parse_string(line) if line.begins_with("{") else null + if not parsed is Dictionary: + if line.contains("ERROR") or line.contains("SCRIPT ERROR") or line.contains("WARNING: ObjectDB instances leaked"): + errors.append(line.strip_edges()) + continue + var record: Dictionary = parsed + var event: String = String(record.get("event", "")) + if event == "match" and record.get("type", "") == "match_started": + started = record + elif event == "match" and record.get("type", "") == "match_ended" and not started.is_empty(): + matches.append({ + "seed": int(record["match"]), "index": int(started.get("index", matches.size())), + "ticks": int(record["tick"]) - int(started["tick"]), + "seconds": (float(record["t_ms"]) - float(started["t_ms"])) / 1000.0, + "winners": record.get("winners", []), "timed_out": bool(record.get("timed_out", false)), + "static_mb": float(record["mem_bytes"]) / 1048576.0, "t_s": float(record["t_ms"]) / 1000.0, + }) + started = {} + elif event == "stop": + stop = record + # Resident memory at each match's end: the last sample before it. + var samples: Array[Vector2] = [] + for line: String in rss_lines: + var parts: PackedStringArray = line.split(" ", false) + if parts.size() == 2 and parts[1].to_int() > 0: + samples.append(Vector2(parts[0].to_float(), parts[1].to_float() / 1024.0)) + var peak: float = 0.0 + for sample: Vector2 in samples: + peak = maxf(peak, sample.y) + for row: Dictionary in matches: + row["rss_mb"] = 0.0 + for sample: Vector2 in samples: + if sample.x <= row["t_s"] + 1.0: + row["rss_mb"] = sample.y + var statics: Array[float] = [] + var rss: Array[float] = [] + for index: int in range(WARMUP_MATCHES, matches.size()): + statics.append(float(matches[index]["static_mb"]) * 1048576.0) + if matches[index]["rss_mb"] > 0.0: + rss.append(float(matches[index]["rss_mb"]) * 1048576.0) + var a: Dictionary = { + "matches": matches, "ended": matches.size(), "errors": errors, "exit_code": exit_code, "stopped": stopped, + "static_slope": slope(statics), "rss_slope": slope(rss), "peak_rss_mb": peak, + "wall_seconds": samples[-1].x if not samples.is_empty() else 0.0, + "server_errors": int(stop.get("errors", -1)), "problems": [], + } + var problems: Array = a["problems"] + if stopped != "": + problems.append("the run was stopped: %s" % stopped) + if matches.size() != expected: + problems.append("%d of %d matches ended" % [matches.size(), expected]) + if exit_code != 0: + problems.append("the server exited %d" % exit_code) + if stop.is_empty(): + problems.append("the server never logged its stop") + elif int(stop.get("errors", 0)) != 0: + problems.append("the server counted %d errors, first: %s" % [stop["errors"], stop.get("first_error", "")]) + if not errors.is_empty(): + problems.append("%d error lines, first: %s" % [errors.size(), errors[0]]) + for row: Dictionary in matches: + if row["winners"].is_empty(): + problems.append("match %d (seed %d) ended with no winner" % [row["index"], row["seed"]]) + if row["seconds"] > match_seconds: + problems.append("match %d (seed %d) took %.0f s, over the %.0f s hang limit" % [row["index"], row["seed"], row["seconds"], match_seconds]) + if matches.size() >= WARMUP_MATCHES + 10: + if a["static_slope"] > MAX_STATIC_GROWTH_PER_MATCH: + problems.append("static memory grows %.0f KiB a match (limit %.0f)" % [a["static_slope"] / 1024.0, MAX_STATIC_GROWTH_PER_MATCH / 1024.0]) + if a["rss_slope"] > MAX_RSS_GROWTH_PER_MATCH: + problems.append("resident memory grows %.0f KiB a match (limit %.0f)" % [a["rss_slope"] / 1024.0, MAX_RSS_GROWTH_PER_MATCH / 1024.0]) + else: + problems.append("only %d matches; the memory trend needs at least %d" % [matches.size(), WARMUP_MATCHES + 10]) + return a + + +static func format(a: Dictionary) -> String: + var lines: PackedStringArray = [] + lines.append("Tidewall long-run stability: %d matches back to back on one server, unthrottled" % a["ended"]) + lines.append(" wall time %.0f s, exit code %d, server errors %d, error lines %d" % [a["wall_seconds"], a["exit_code"], a["server_errors"], a["errors"].size()]) + lines.append(" memory after the first %d matches: static %+.1f KiB a match, resident %+.1f KiB a match; peak resident %.0f MB" % [ + WARMUP_MATCHES, a["static_slope"] / 1024.0, a["rss_slope"] / 1024.0, a["peak_rss_mb"]]) + lines.append("") + lines.append(" # seed ticks wall s winner static MB resident MB") + for row: Dictionary in a["matches"]: + lines.append(" %-3d %-20d %6d %7.1f %7s %10.1f %12.1f%s" % [row["index"], row["seed"], row["ticks"], row["seconds"], + "yes" if not row["winners"].is_empty() else "NO", row["static_mb"], row["rss_mb"], " (tide's clock)" if row["timed_out"] else ""]) + lines.append("") + if a["problems"].is_empty(): + lines.append("stability: ok") + else: + for problem: String in a["problems"]: + lines.append("stability FAIL: %s" % problem) + return "\n".join(lines) + "\n" diff --git a/test/server/stability_report.gd.uid b/test/server/stability_report.gd.uid new file mode 100644 index 0000000..39648b8 --- /dev/null +++ b/test/server/stability_report.gd.uid @@ -0,0 +1 @@ +uid://b8e85xf8plubp diff --git a/test/server/test_stability_report.gd b/test/server/test_stability_report.gd new file mode 100644 index 0000000..df33d22 --- /dev/null +++ b/test/server/test_stability_report.gd @@ -0,0 +1,64 @@ +extends TestCase + +## The long-run stability report (H4): what counts as a clean run of back-to-back +## matches, and that a memory trend, a hang, an error or a missing match fails it. + + +func _log(matches: int, static_growth: float, seconds: float = 20.0, winners: bool = true) -> PackedStringArray: + var lines: PackedStringArray = ["Godot Engine v4.7.2.stable.official - https://godotengine.org"] + var tick: int = 0 + var t_ms: float = 0.0 + for index: int in matches: + if index > 0: + lines.append(JSON.stringify({"event": "match", "type": "match_started", "match": 1000 + index, "index": index, "tick": tick, "t_ms": t_ms, "mem_bytes": 70000000})) + tick += 25000 + t_ms += seconds * 1000.0 + lines.append(JSON.stringify({"event": "match", "type": "match_ended", "match": 1000 + index, "winners": [-3] if winners else [], "timed_out": false, + "tick": tick, "t_ms": t_ms, "mem_bytes": 70000000 + roundi(static_growth * index)})) + lines.append(JSON.stringify({"event": "stop", "exit_code": 0, "errors": 0, "tick": tick, "t_ms": t_ms})) + return lines + + +func _rss(matches: int, growth_kb: float, seconds: float = 20.0) -> PackedStringArray: + var lines: PackedStringArray = [] + for index: int in matches * 4 + 1: + var at: float = index * seconds / 4.0 + lines.append("%d %d" % [roundi(at), 200000 + roundi(growth_kb * at / seconds)]) + return lines + + +func test_slope_is_least_squares() -> void: + assert_eq(StabilityReport.slope([1.0, 3.0, 5.0] as Array[float]), 2.0) + assert_eq(StabilityReport.slope([4.0, 4.0, 4.0, 4.0] as Array[float]), 0.0) + assert_eq(StabilityReport.slope([7.0] as Array[float]), 0.0) + + +func test_a_clean_flat_run_passes() -> void: + var a: Dictionary = StabilityReport.analyse(_log(50, 1000.0), _rss(50, 10.0), 0, "", 50, 240.0) + assert_eq(a["problems"], [], "%s" % [a["problems"]]) + assert_eq(a["ended"], 50) + assert_eq(a["matches"][0]["ticks"], 25000, "the first match counts from the start") + assert_near(a["static_slope"], 1000.0, 1.0) + assert_true(a["rss_slope"] > 0.0 and a["rss_slope"] < 20.0 * 1024.0) + assert_true(StabilityReport.format(a).contains("stability: ok")) + + +func test_growth_hangs_errors_and_missing_matches_fail() -> void: + var leak: Dictionary = StabilityReport.analyse(_log(50, 512.0 * 1024.0), _rss(50, 0.0), 0, "", 50, 240.0) + assert_eq(leak["problems"].size(), 1, "%s" % [leak["problems"]]) + assert_true(String(leak["problems"][0]).begins_with("static memory grows 512 KiB")) + var rss: Dictionary = StabilityReport.analyse(_log(50, 0.0), _rss(50, 4096.0), 0, "", 50, 240.0) + assert_true(String(rss["problems"][0]).begins_with("resident memory grows"), "%s" % [rss["problems"]]) + var slow: Dictionary = StabilityReport.analyse(_log(20, 0.0, 300.0), _rss(20, 0.0, 300.0), 0, "", 20, 240.0) + assert_eq(slow["problems"].size(), 20, "every match over the hang limit") + var lines: PackedStringArray = _log(20, 0.0) + lines.insert(3, "SCRIPT ERROR: Invalid call. Nonexistent function 'x' in base 'Nil'.") + var broken: Dictionary = StabilityReport.analyse(lines, _rss(20, 0.0), 1, "", 20, 240.0) + var text: String = StabilityReport.format(broken) + assert_true(text.contains("the server exited 1")) + assert_true(text.contains("1 error lines, first: SCRIPT ERROR")) + var short: Dictionary = StabilityReport.analyse(_log(30, 0.0, 20.0, false), _rss(30, 0.0), 0, "over the 7320 s limit: a hang", 50, 240.0) + var short_text: String = StabilityReport.format(short) + assert_true(short_text.contains("the run was stopped: over the 7320 s limit")) + assert_true(short_text.contains("30 of 50 matches ended")) + assert_true(short_text.contains("ended with no winner")) diff --git a/test/server/test_stability_report.gd.uid b/test/server/test_stability_report.gd.uid new file mode 100644 index 0000000..2cbb15b --- /dev/null +++ b/test/server/test_stability_report.gd.uid @@ -0,0 +1 @@ +uid://c26lqrxplefwk diff --git a/test/server/test_tick_times.gd b/test/server/test_tick_times.gd new file mode 100644 index 0000000..dc5cb6c --- /dev/null +++ b/test/server/test_tick_times.gd @@ -0,0 +1,40 @@ +extends TestCase + +## H4: tick timing in fixed memory, and the server's launchers keeping the allocator to +## one arena so resident memory doesn't grow match after match. + + +func test_statistics_match_a_sorted_list() -> void: + var times: TickTimes = TickTimes.new() + var values: PackedInt64Array = PackedInt64Array() + var rng: RandomNumberGenerator = RandomNumberGenerator.new() + rng.seed = 4 + for index: int in 5000: + var usec: int = rng.randi_range(200, 3000) if index % 500 != 0 else rng.randi_range(70000, 2000000) + times.add(usec) + values.append(usec) + values.sort() + for fraction: float in [0.0, 0.01, 0.5, 0.95, 0.99, 0.999, 1.0]: + assert_eq(times.percentile(fraction), ServerLoop._percentile(values, fraction), "percentile %.3f" % fraction) + var stats: Dictionary = times.stats(ServerLoop.TICK_BUDGET_USEC) + assert_eq(stats["ticks"], 5000) + assert_eq(stats["min_us"], values[0]) + assert_eq(stats["max_us"], values[-1]) + assert_eq(stats["over_budget"], 10, "the ten slow ticks") + assert_eq(TickTimes.new().stats(1000), {"ticks": 0}) + + +func test_memory_stays_fixed_past_the_slow_tick_cap() -> void: + var times: TickTimes = TickTimes.new() + for index: int in TickTimes.MAX_SLOW + 100: + times.add(TickTimes.HISTOGRAM_USEC + index) + assert_eq(times._slow.size(), TickTimes.MAX_SLOW) + assert_eq(times.size(), TickTimes.MAX_SLOW + 100) + assert_eq(times.percentile(1.0), TickTimes.HISTOGRAM_USEC + TickTimes.MAX_SLOW + 99, "the slowest are kept") + assert_eq(times.over(12000), TickTimes.MAX_SLOW + 100) + + +func test_server_launchers_use_one_malloc_arena() -> void: + for path: String in ["res://packaging/server/tidewall-server", "res://packaging/dist/tidewall-server", "res://tools/godot.sh"]: + var text: String = FileAccess.get_file_as_string(path) + assert_true(text.contains('export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}"'), "%s sets MALLOC_ARENA_MAX" % path) diff --git a/test/server/test_tick_times.gd.uid b/test/server/test_tick_times.gd.uid new file mode 100644 index 0000000..77d9eae --- /dev/null +++ b/test/server/test_tick_times.gd.uid @@ -0,0 +1 @@ +uid://2ib5r4fuuxpe diff --git a/tools/godot.sh b/tools/godot.sh index a0a45f5..47ce4d5 100755 --- a/tools/godot.sh +++ b/tools/godot.sh @@ -63,4 +63,12 @@ if [[ "${TIDEWALL_GODOT_DISPLAY:-0}" != "1" ]]; then unset DISPLAY WAYLAND_DISPLAY fi +# A dedicated server gets one malloc arena, as the packaged launchers give it: see +# packaging/server/tidewall-server (H4). +for arg in "$@"; do + if [[ "$arg" == "--server" ]]; then + export MALLOC_ARENA_MAX="${MALLOC_ARENA_MAX:-1}" + fi +done + exec "$binary" "${args[@]}" "$@" diff --git a/tools/stability.sh b/tools/stability.sh new file mode 100755 index 0000000..6fccdb1 --- /dev/null +++ b/tools/stability.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# H4: long-run stability. One dedicated server process plays STABILITY_MATCHES (default +# 50) whole bot matches back to back, unthrottled (accelerated time: the server never +# waits for the clock), from seed STABILITY_SEED (default 4242), under nice. Each match +# gets its own seed from the server's. The process's resident memory is sampled every +# few seconds. tools/stability_report.gd then checks the log and samples: see +# test/server/stability_report.gd for what passes. About 25 minutes on this host. +# +# tools/stability.sh +# STABILITY_MATCHES=5 tools/stability.sh a quick look (too few to pass) +# +# If the host's available memory falls under 2 GB, the run is stopped and fails. +set -uo pipefail +repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo" +matches="${STABILITY_MATCHES:-50}" +seed="${STABILITY_SEED:-4242}" +out="${STABILITY_DIR:-$repo/build/stability}" +# A match is about 25,000 ticks; unthrottled that is well under a minute here. A hang is +# anything over this many seconds of wall time per match. +per_match_limit="${STABILITY_MATCH_SECONDS:-240}" +limit=$(( matches * per_match_limit + 120 )) +rm -rf "$out" +mkdir -p "$out" + +nice -n 15 tools/godot.sh -- --server --seed="$seed" --bots=24 --matches="$matches" --unthrottled \ + --log-every=3000 >"$out/server.log" 2>&1 & +pid=$! +started=$(date +%s) +stopped="" +: >"$out/rss.txt" +while kill -0 "$pid" 2>/dev/null; do + # The engine runs as a child of tools/godot.sh; sample the biggest process in the group. + rss=0 + for p in "$pid" $(pgrep -P "$pid" 2>/dev/null); do + r=$(awk '/^VmRSS:/ { print $2 }' "/proc/$p/status" 2>/dev/null || echo 0) + (( ${r:-0} > rss )) && rss=${r:-0} + done + echo "$(( $(date +%s) - started )) $rss" >>"$out/rss.txt" + avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo) + if (( avail < 2048 )); then + stopped="host memory low (${avail} MB available)" + elif (( $(date +%s) - started > limit )); then + stopped="over the ${limit} s limit: a hang" + fi + if [[ -n "$stopped" ]]; then + pkill -TERM -P "$pid" 2>/dev/null; kill -TERM "$pid" 2>/dev/null + sleep 3 + pkill -KILL -P "$pid" 2>/dev/null; kill -KILL "$pid" 2>/dev/null + break + fi + sleep 5 +done +wait "$pid" 2>/dev/null +code=$? +echo "$code" >"$out/exit_code.txt" +[[ -n "$stopped" ]] && echo "$stopped" >"$out/stopped.txt" +tools/godot.sh --script res://tools/stability_report.gd -- --dir="$out" --matches="$matches" \ + --match-seconds="$per_match_limit" diff --git a/tools/stability_report.gd b/tools/stability_report.gd new file mode 100644 index 0000000..3e9ae13 --- /dev/null +++ b/tools/stability_report.gd @@ -0,0 +1,35 @@ +## Reads a tools/stability.sh run (server.log, rss.txt, exit_code.txt, stopped.txt) and +## judges it with StabilityReport. Writes report.txt beside them. Exits 1 on a failure. +## +## tools/godot.sh --script res://tools/stability_report.gd -- --dir=PATH --matches=50 +extends SceneTree + + +func _init() -> void: + var dir_path: String = "" + var matches: int = 50 + var match_seconds: float = 240.0 + for arg: String in OS.get_cmdline_user_args(): + if arg.begins_with("--dir="): + dir_path = arg.trim_prefix("--dir=") + elif arg.begins_with("--matches="): + matches = arg.trim_prefix("--matches=").to_int() + elif arg.begins_with("--match-seconds="): + match_seconds = arg.trim_prefix("--match-seconds=").to_float() + if dir_path == "": + printerr("stability: give --dir=PATH") + quit(2) + return + var log_lines: PackedStringArray = FileAccess.get_file_as_string(dir_path.path_join("server.log")).split("\n", false) + var rss: PackedStringArray = FileAccess.get_file_as_string(dir_path.path_join("rss.txt")).split("\n", false) + var code: int = FileAccess.get_file_as_string(dir_path.path_join("exit_code.txt")).strip_edges().to_int() + var stopped: String = "" + if FileAccess.file_exists(dir_path.path_join("stopped.txt")): + stopped = FileAccess.get_file_as_string(dir_path.path_join("stopped.txt")).strip_edges() + var a: Dictionary = StabilityReport.analyse(log_lines, rss, code, stopped, matches, match_seconds) + var text: String = StabilityReport.format(a) + var file: FileAccess = FileAccess.open(dir_path.path_join("report.txt"), FileAccess.WRITE) + file.store_string(text) + file.close() + print(text) + quit(0 if a["problems"].is_empty() else 1) diff --git a/tools/stability_report.gd.uid b/tools/stability_report.gd.uid new file mode 100644 index 0000000..15631e0 --- /dev/null +++ b/tools/stability_report.gd.uid @@ -0,0 +1 @@ +uid://cganord5usyws