#!/usr/bin/env bash # H4: long-run stability. One dedicated server process plays STABILITY_MATCHES (default # 50) whole bot matches back to back, unthrottled (accelerated time: the server never # waits for the clock), from seed STABILITY_SEED (default 4242), under nice. Each match # gets its own seed from the server's. The process's resident memory is sampled every # few seconds. tools/stability_report.gd then checks the log and samples: see # test/server/stability_report.gd for what passes. About 25 minutes on this host. # # tools/stability.sh # STABILITY_MATCHES=5 tools/stability.sh a quick look (too few to pass) # # If the host's available memory falls under 2 GB, the run is stopped and fails. set -uo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo" matches="${STABILITY_MATCHES:-50}" seed="${STABILITY_SEED:-4242}" out="${STABILITY_DIR:-$repo/build/stability}" # A match is about 25,000 ticks; unthrottled that is well under a minute here. A hang is # anything over this many seconds of wall time per match. per_match_limit="${STABILITY_MATCH_SECONDS:-240}" limit=$(( matches * per_match_limit + 120 )) rm -rf "$out" mkdir -p "$out" nice -n 15 tools/godot.sh -- --server --seed="$seed" --bots=24 --matches="$matches" --unthrottled \ --log-every=3000 >"$out/server.log" 2>&1 & pid=$! started=$(date +%s) stopped="" : >"$out/rss.txt" while kill -0 "$pid" 2>/dev/null; do # The engine runs as a child of tools/godot.sh; sample the biggest process in the group. rss=0 for p in "$pid" $(pgrep -P "$pid" 2>/dev/null); do r=$(awk '/^VmRSS:/ { print $2 }' "/proc/$p/status" 2>/dev/null || echo 0) (( ${r:-0} > rss )) && rss=${r:-0} done echo "$(( $(date +%s) - started )) $rss" >>"$out/rss.txt" avail=$(awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo) if (( avail < 2048 )); then stopped="host memory low (${avail} MB available)" elif (( $(date +%s) - started > limit )); then stopped="over the ${limit} s limit: a hang" fi if [[ -n "$stopped" ]]; then pkill -TERM -P "$pid" 2>/dev/null; kill -TERM "$pid" 2>/dev/null sleep 3 pkill -KILL -P "$pid" 2>/dev/null; kill -KILL "$pid" 2>/dev/null break fi sleep 5 done wait "$pid" 2>/dev/null code=$? echo "$code" >"$out/exit_code.txt" [[ -n "$stopped" ]] && echo "$stopped" >"$out/stopped.txt" tools/godot.sh --script res://tools/stability_report.gd -- --dir="$out" --matches="$matches" \ --match-seconds="$per_match_limit"