make soak runs a real server and 24 headless clients in three processes for two whole matches in real time. Each client reaches the server through its own UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs unchanged over a bad link. SoakPlayer drives each client from what a client knows: it drops, glides, heads for high ground, harvests and tries to build. tools/soak_report.gd checks world hashes against the server's, errors, unreadable packets, every client playing every match with its connection intact, memory between matches, and the p99 tick with the network's share. make soak-selfcheck proves a drifted client world is caught. Server changes: --matches=N, --world-hash-every=N, tick timing that includes after_tick, and memory in match and progress lines. What the soak found and fixed: - Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the 12 ms budget, against a 2.8 ms simulation. Encoded records are now shared between clients for the tick, unchanged records skip the field diff, and clients are spread over the snapshot interval by handle. p99 is 7.3 ms. - ENet peers had a timeout limit of 0, which gives up on a peer as soon as any reliable message has waited the minimum timeout. Now ENet's default, 32. Not fixed: one or two clients still lose their connection about 15 s into the second match's drop in each run (see the issue), so the soak still fails and N8 stays open. Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
120 lines
4.3 KiB
Bash
Executable File
120 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# The 24-client network soak (issue N8). Starts a real dedicated server process and
|
|
# SOAK_PROCS client processes, which between them run SOAK_CLIENTS headless clients.
|
|
# Every client reaches the server through its own UdpRelay, which adds SOAK_LATENCY ms
|
|
# of delay, +/- SOAK_JITTER ms of jitter and SOAK_LOSS of loss each way. The clients
|
|
# are played by SoakPlayer: they drop, glide, run for high ground, harvest and build.
|
|
# The server plays SOAK_MATCHES whole matches in real time and stops, which ends the
|
|
# clients, and tools/soak_report.gd decides whether it all passed: see that file.
|
|
#
|
|
# A full match is about 15 minutes of real time, so the default run (two matches) takes
|
|
# about half an hour. Everything runs under nice. If the host's available memory falls
|
|
# under 2 GB the run is stopped and fails, rather than squeezing other work.
|
|
#
|
|
# tools/soak.sh SOAK_MATCHES=1 SOAK_CLIENTS=8 ... to scale it
|
|
# SOAK_SELFCHECK=1 tools/soak.sh a minute's run with one client's world tampered
|
|
# with, which must be caught (make soak-selfcheck)
|
|
set -uo pipefail
|
|
|
|
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$repo"
|
|
selfcheck="${SOAK_SELFCHECK:-0}"
|
|
clients="${SOAK_CLIENTS:-24}"
|
|
procs="${SOAK_PROCS:-3}"
|
|
matches="${SOAK_MATCHES:-2}"
|
|
latency="${SOAK_LATENCY:-60}"
|
|
jitter="${SOAK_JITTER:-20}"
|
|
loss="${SOAK_LOSS:-0.02}"
|
|
seed="${SOAK_SEED:-7}"
|
|
port="${SOAK_PORT:-24680}"
|
|
hash_every="${SOAK_HASH_EVERY:-300}"
|
|
out="$repo/build/soak"
|
|
server_args=(--matches="$matches")
|
|
tamper=()
|
|
if [[ "$selfcheck" == "1" ]]; then
|
|
clients=2
|
|
procs=1
|
|
matches=0
|
|
hash_every=30
|
|
out="$repo/build/soak-selfcheck"
|
|
# Lobby and countdown take 900 ticks; then a little under a minute of the match.
|
|
server_args=(--ticks=2400)
|
|
tamper=(--tamper=1)
|
|
fi
|
|
limit_seconds=$(( matches > 0 ? matches * 1500 + 300 : 300 ))
|
|
rm -rf "$out"
|
|
mkdir -p "$out"
|
|
pids=()
|
|
|
|
stop_all() {
|
|
for pid in "${pids[@]}"; do kill "$pid" 2>/dev/null || true; done
|
|
}
|
|
trap stop_all EXIT
|
|
|
|
available_mb() {
|
|
awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo
|
|
}
|
|
|
|
echo "soak: $clients clients in $procs processes, $matches matches, ${latency} ms +/- ${jitter} ms, loss $loss, seed $seed (logs in ${out#$repo/})"
|
|
nice -n 15 tools/godot.sh -- --server --listen --no-lan --port="$port" --seed="$seed" --bots=1 \
|
|
--log-every=900 --world-hash-every="$hash_every" "${server_args[@]}" >"$out/server.log" 2>&1 &
|
|
server=$!
|
|
pids+=("$server")
|
|
sleep 3
|
|
logs=()
|
|
per=$(( (clients + procs - 1) / procs ))
|
|
first=0
|
|
for (( index = 0; index < procs && first < clients; index++ )); do
|
|
count=$(( clients - first < per ? clients - first : per ))
|
|
log="$out/clients_$index.log"
|
|
logs+=("$log")
|
|
nice -n 15 tools/godot.sh --script res://tools/soak_clients.gd -- --server="127.0.0.1:$port" \
|
|
--clients="$count" --first="$first" --latency="$latency" --jitter="$jitter" --loss="$loss" \
|
|
--seed="$seed" --hash-every="$hash_every" --seconds="$limit_seconds" "${tamper[@]}" >"$log" 2>&1 &
|
|
pids+=("$!")
|
|
first=$(( first + count ))
|
|
done
|
|
|
|
# Wait for everything to finish, watching the host's memory and the time limit.
|
|
started=$SECONDS
|
|
low_memory=0
|
|
while :; do
|
|
running=0
|
|
for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && running=1; done
|
|
[[ "$running" -eq 0 ]] && break
|
|
if (( $(available_mb) < 2048 )); then
|
|
low_memory=1
|
|
echo "soak: the host has under 2 GB available; stopping" >&2
|
|
stop_all
|
|
break
|
|
fi
|
|
if (( SECONDS - started > limit_seconds )); then
|
|
echo "soak: still running after ${limit_seconds}s; stopping" >&2
|
|
stop_all
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
wait 2>/dev/null
|
|
trap - EXIT
|
|
[[ "$low_memory" -eq 0 ]] || exit 1
|
|
|
|
joined="$(IFS=,; echo "${logs[*]}")"
|
|
report="$(tools/godot.sh --script res://tools/soak_report.gd -- --server-log="$out/server.log" \
|
|
--client-logs="$joined" --clients="$clients" --matches="$matches" 2>&1 | grep -v '^Godot Engine')"
|
|
echo "$report" >"$out/report.txt"
|
|
|
|
if [[ "$selfcheck" == "1" ]]; then
|
|
if grep -q "client 1's world differs" <<<"$report" && ! grep -q "client 0's world differs" <<<"$report" \
|
|
&& grep -q '"hashes_compared":[1-9]' <<<"$report"; then
|
|
echo "soak-selfcheck: ok (the tampered client was caught, the other wasn't)"
|
|
exit 0
|
|
fi
|
|
echo "soak-selfcheck: FAIL, the tampered world wasn't caught as it should be" >&2
|
|
echo "$report" >&2
|
|
exit 1
|
|
fi
|
|
echo "$report"
|
|
grep -q '^soak FAIL' <<<"$report" && exit 1
|
|
echo "soak: ok"
|