#!/usr/bin/env bash # The 24-client network soak (issue N8). Starts a real dedicated server process and # SOAK_PROCS client processes, which between them run SOAK_CLIENTS headless clients. # Every client reaches the server through its own UdpRelay, which adds SOAK_LATENCY ms # of delay, +/- SOAK_JITTER ms of jitter and SOAK_LOSS of loss each way. The clients # are played by SoakPlayer: they drop, glide, run for high ground, harvest and build. # The server plays SOAK_MATCHES whole matches in real time and stops, which ends the # clients, and tools/soak_report.gd decides whether it all passed: see that file. # # A full match is about 15 minutes of real time, so the default run (two matches) takes # about half an hour. Everything runs under nice. If the host's available memory falls # under 2 GB the run is stopped and fails, rather than squeezing other work. # # tools/soak.sh SOAK_MATCHES=1 SOAK_CLIENTS=8 ... to scale it # SOAK_SELFCHECK=1 tools/soak.sh a minute's run with one client's world tampered # with, which must be caught (make soak-selfcheck) set -uo pipefail repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$repo" selfcheck="${SOAK_SELFCHECK:-0}" clients="${SOAK_CLIENTS:-24}" procs="${SOAK_PROCS:-3}" matches="${SOAK_MATCHES:-2}" latency="${SOAK_LATENCY:-60}" jitter="${SOAK_JITTER:-20}" loss="${SOAK_LOSS:-0.02}" seed="${SOAK_SEED:-7}" port="${SOAK_PORT:-24680}" hash_every="${SOAK_HASH_EVERY:-300}" out="$repo/build/soak" server_args=(--matches="$matches") tamper=() if [[ "$selfcheck" == "1" ]]; then clients=2 procs=1 matches=0 hash_every=30 out="$repo/build/soak-selfcheck" # Lobby and countdown take 900 ticks; then a little under a minute of the match. server_args=(--ticks=2400) tamper=(--tamper=1) fi limit_seconds=$(( matches > 0 ? matches * 1500 + 300 : 300 )) rm -rf "$out" mkdir -p "$out" pids=() stop_all() { for pid in "${pids[@]}"; do kill "$pid" 2>/dev/null || true; done } trap stop_all EXIT available_mb() { awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo } echo "soak: $clients clients in $procs processes, $matches matches, ${latency} ms +/- ${jitter} ms, loss $loss, seed $seed (logs in ${out#$repo/})" nice -n 15 tools/godot.sh -- --server --listen --no-lan --port="$port" --seed="$seed" --bots=1 \ --log-every=900 --world-hash-every="$hash_every" "${server_args[@]}" >"$out/server.log" 2>&1 & server=$! pids+=("$server") sleep 3 logs=() per=$(( (clients + procs - 1) / procs )) first=0 for (( index = 0; index < procs && first < clients; index++ )); do count=$(( clients - first < per ? clients - first : per )) log="$out/clients_$index.log" logs+=("$log") nice -n 15 tools/godot.sh --script res://tools/soak_clients.gd -- --server="127.0.0.1:$port" \ --clients="$count" --first="$first" --latency="$latency" --jitter="$jitter" --loss="$loss" \ --seed="$seed" --hash-every="$hash_every" --seconds="$limit_seconds" "${tamper[@]}" >"$log" 2>&1 & pids+=("$!") first=$(( first + count )) done # Wait for everything to finish, watching the host's memory and the time limit. started=$SECONDS low_memory=0 while :; do running=0 for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && running=1; done [[ "$running" -eq 0 ]] && break if (( $(available_mb) < 2048 )); then low_memory=1 echo "soak: the host has under 2 GB available; stopping" >&2 stop_all break fi if (( SECONDS - started > limit_seconds )); then echo "soak: still running after ${limit_seconds}s; stopping" >&2 stop_all break fi sleep 5 done wait 2>/dev/null trap - EXIT [[ "$low_memory" -eq 0 ]] || exit 1 joined="$(IFS=,; echo "${logs[*]}")" report="$(tools/godot.sh --script res://tools/soak_report.gd -- --server-log="$out/server.log" \ --client-logs="$joined" --clients="$clients" --matches="$matches" 2>&1 | grep -v '^Godot Engine')" echo "$report" >"$out/report.txt" if [[ "$selfcheck" == "1" ]]; then if grep -q "client 1's world differs" <<<"$report" && ! grep -q "client 0's world differs" <<<"$report" \ && grep -q '"hashes_compared":[1-9]' <<<"$report"; then echo "soak-selfcheck: ok (the tampered client was caught, the other wasn't)" exit 0 fi echo "soak-selfcheck: FAIL, the tampered world wasn't caught as it should be" >&2 echo "$report" >&2 exit 1 fi echo "$report" grep -q '^soak FAIL' <<<"$report" && exit 1 echo "soak: ok"