N8: 24-client network soak, and the server fixes it found
make soak runs a real server and 24 headless clients in three processes for two whole matches in real time. Each client reaches the server through its own UdpRelay, which adds 60 ms, +/-20 ms jitter and 2% loss each way, so ENet runs unchanged over a bad link. SoakPlayer drives each client from what a client knows: it drops, glides, heads for high ground, harvests and tries to build. tools/soak_report.gd checks world hashes against the server's, errors, unreadable packets, every client playing every match with its connection intact, memory between matches, and the p99 tick with the network's share. make soak-selfcheck proves a drifted client world is caught. Server changes: --matches=N, --world-hash-every=N, tick timing that includes after_tick, and memory in match and progress lines. What the soak found and fixed: - Snapshot building for 24 clients took the p99 tick to 13.8 ms, over the 12 ms budget, against a 2.8 ms simulation. Encoded records are now shared between clients for the tick, unchanged records skip the field diff, and clients are spread over the snapshot interval by handle. p99 is 7.3 ms. - ENet peers had a timeout limit of 0, which gives up on a peer as soon as any reliable message has waited the minimum timeout. Now ENet's default, 32. Not fixed: one or two clients still lose their connection about 15 s into the second match's drop in each run (see the issue), so the soak still fails and N8 stays open. Refs #39 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# The 24-client network soak (issue N8). Starts a real dedicated server process and
|
||||
# SOAK_PROCS client processes, which between them run SOAK_CLIENTS headless clients.
|
||||
# Every client reaches the server through its own UdpRelay, which adds SOAK_LATENCY ms
|
||||
# of delay, +/- SOAK_JITTER ms of jitter and SOAK_LOSS of loss each way. The clients
|
||||
# are played by SoakPlayer: they drop, glide, run for high ground, harvest and build.
|
||||
# The server plays SOAK_MATCHES whole matches in real time and stops, which ends the
|
||||
# clients, and tools/soak_report.gd decides whether it all passed: see that file.
|
||||
#
|
||||
# A full match is about 15 minutes of real time, so the default run (two matches) takes
|
||||
# about half an hour. Everything runs under nice. If the host's available memory falls
|
||||
# under 2 GB the run is stopped and fails, rather than squeezing other work.
|
||||
#
|
||||
# tools/soak.sh SOAK_MATCHES=1 SOAK_CLIENTS=8 ... to scale it
|
||||
# SOAK_SELFCHECK=1 tools/soak.sh a minute's run with one client's world tampered
|
||||
# with, which must be caught (make soak-selfcheck)
|
||||
set -uo pipefail
|
||||
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$repo"
|
||||
selfcheck="${SOAK_SELFCHECK:-0}"
|
||||
clients="${SOAK_CLIENTS:-24}"
|
||||
procs="${SOAK_PROCS:-3}"
|
||||
matches="${SOAK_MATCHES:-2}"
|
||||
latency="${SOAK_LATENCY:-60}"
|
||||
jitter="${SOAK_JITTER:-20}"
|
||||
loss="${SOAK_LOSS:-0.02}"
|
||||
seed="${SOAK_SEED:-7}"
|
||||
port="${SOAK_PORT:-24680}"
|
||||
hash_every="${SOAK_HASH_EVERY:-300}"
|
||||
out="$repo/build/soak"
|
||||
server_args=(--matches="$matches")
|
||||
tamper=()
|
||||
if [[ "$selfcheck" == "1" ]]; then
|
||||
clients=2
|
||||
procs=1
|
||||
matches=0
|
||||
hash_every=30
|
||||
out="$repo/build/soak-selfcheck"
|
||||
# Lobby and countdown take 900 ticks; then a little under a minute of the match.
|
||||
server_args=(--ticks=2400)
|
||||
tamper=(--tamper=1)
|
||||
fi
|
||||
limit_seconds=$(( matches > 0 ? matches * 1500 + 300 : 300 ))
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
pids=()
|
||||
|
||||
stop_all() {
|
||||
for pid in "${pids[@]}"; do kill "$pid" 2>/dev/null || true; done
|
||||
}
|
||||
trap stop_all EXIT
|
||||
|
||||
available_mb() {
|
||||
awk '/^MemAvailable:/ { print int($2 / 1024) }' /proc/meminfo
|
||||
}
|
||||
|
||||
echo "soak: $clients clients in $procs processes, $matches matches, ${latency} ms +/- ${jitter} ms, loss $loss, seed $seed (logs in ${out#$repo/})"
|
||||
nice -n 15 tools/godot.sh -- --server --listen --no-lan --port="$port" --seed="$seed" --bots=1 \
|
||||
--log-every=900 --world-hash-every="$hash_every" "${server_args[@]}" >"$out/server.log" 2>&1 &
|
||||
server=$!
|
||||
pids+=("$server")
|
||||
sleep 3
|
||||
logs=()
|
||||
per=$(( (clients + procs - 1) / procs ))
|
||||
first=0
|
||||
for (( index = 0; index < procs && first < clients; index++ )); do
|
||||
count=$(( clients - first < per ? clients - first : per ))
|
||||
log="$out/clients_$index.log"
|
||||
logs+=("$log")
|
||||
nice -n 15 tools/godot.sh --script res://tools/soak_clients.gd -- --server="127.0.0.1:$port" \
|
||||
--clients="$count" --first="$first" --latency="$latency" --jitter="$jitter" --loss="$loss" \
|
||||
--seed="$seed" --hash-every="$hash_every" --seconds="$limit_seconds" "${tamper[@]}" >"$log" 2>&1 &
|
||||
pids+=("$!")
|
||||
first=$(( first + count ))
|
||||
done
|
||||
|
||||
# Wait for everything to finish, watching the host's memory and the time limit.
|
||||
started=$SECONDS
|
||||
low_memory=0
|
||||
while :; do
|
||||
running=0
|
||||
for pid in "${pids[@]}"; do kill -0 "$pid" 2>/dev/null && running=1; done
|
||||
[[ "$running" -eq 0 ]] && break
|
||||
if (( $(available_mb) < 2048 )); then
|
||||
low_memory=1
|
||||
echo "soak: the host has under 2 GB available; stopping" >&2
|
||||
stop_all
|
||||
break
|
||||
fi
|
||||
if (( SECONDS - started > limit_seconds )); then
|
||||
echo "soak: still running after ${limit_seconds}s; stopping" >&2
|
||||
stop_all
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
wait 2>/dev/null
|
||||
trap - EXIT
|
||||
[[ "$low_memory" -eq 0 ]] || exit 1
|
||||
|
||||
joined="$(IFS=,; echo "${logs[*]}")"
|
||||
report="$(tools/godot.sh --script res://tools/soak_report.gd -- --server-log="$out/server.log" \
|
||||
--client-logs="$joined" --clients="$clients" --matches="$matches" 2>&1 | grep -v '^Godot Engine')"
|
||||
echo "$report" >"$out/report.txt"
|
||||
|
||||
if [[ "$selfcheck" == "1" ]]; then
|
||||
if grep -q "client 1's world differs" <<<"$report" && ! grep -q "client 0's world differs" <<<"$report" \
|
||||
&& grep -q '"hashes_compared":[1-9]' <<<"$report"; then
|
||||
echo "soak-selfcheck: ok (the tampered client was caught, the other wasn't)"
|
||||
exit 0
|
||||
fi
|
||||
echo "soak-selfcheck: FAIL, the tampered world wasn't caught as it should be" >&2
|
||||
echo "$report" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$report"
|
||||
grep -q '^soak FAIL' <<<"$report" && exit 1
|
||||
echo "soak: ok"
|
||||
@@ -0,0 +1,276 @@
|
||||
## Runs several headless network clients in one process for the soak test (make soak).
|
||||
## Each client reaches the server through its own UdpRelay, which delays, jitters and
|
||||
## loses datagrams both ways, and is played by a SoakPlayer.
|
||||
##
|
||||
## tools/godot.sh --script res://tools/soak_clients.gd -- --server=HOST:PORT
|
||||
## [--clients=N] [--first=N] [--latency=MS] [--jitter=MS] [--loss=FRACTION]
|
||||
## [--seed=N] [--hash-every=N] [--seconds=N] [--tamper=N]
|
||||
##
|
||||
## --tamper=N knocks a point off one prop's health in client N's world just before its
|
||||
## first hash once the match is under way, so `make soak-selfcheck` can prove that a
|
||||
## client whose world has drifted is caught.
|
||||
##
|
||||
## Prints JSON lines:
|
||||
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
|
||||
## the snapshots where the server logs its own (--world-hash-every on the server, the
|
||||
## same N as --hash-every here), when the client's world has caught up exactly.
|
||||
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
|
||||
## for a match it played, once the match is over for it.
|
||||
## {"event":"memory", "match", "mem_bytes"}: this process's memory as each match began.
|
||||
## {"event":"soak_client", ...}: one per client at the end.
|
||||
## {"event":"soak_process", ...}: frame times, totals and errors, last.
|
||||
## It stops when every client's connection has closed, or after --seconds.
|
||||
##
|
||||
## Making a match's island takes most of a second. A real player does it once, but a
|
||||
## dozen clients here would all do it in the same frame and starve every connection of
|
||||
## service for many seconds. So a MATCH notice that brings a new match is held back,
|
||||
## with everything after it for that client, until the driver has made that client's
|
||||
## world, and the driver makes at most one world a frame.
|
||||
## Exits 0, or 1 if any engine or script error happened, or 2 on a bad command line.
|
||||
extends SceneTree
|
||||
|
||||
const FPS: int = 60
|
||||
|
||||
var _clients: Array[Dictionary] = []
|
||||
var _errors: ErrorCounter = ErrorCounter.new()
|
||||
var _hash_every: int = 300
|
||||
var _deadline_msec: int = 0
|
||||
var _frame_usec: PackedInt64Array = PackedInt64Array()
|
||||
var _memory_logged: Dictionary = {}
|
||||
var _started_msec: int = 0
|
||||
var _done: bool = false
|
||||
var _tamper: int = -1
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
OS.add_logger(_errors)
|
||||
var options: Dictionary = {
|
||||
"server": "", "clients": "12", "first": "0", "latency": "60", "jitter": "20",
|
||||
"loss": "0.02", "seed": "1", "hash-every": "300", "seconds": "3600", "tamper": "-1",
|
||||
}
|
||||
for arg: String in OS.get_cmdline_user_args():
|
||||
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
||||
if parts.size() != 2 or not options.has(parts[0]):
|
||||
printerr("soak_clients: unknown argument '%s'" % arg)
|
||||
quit(2)
|
||||
return
|
||||
options[parts[0]] = parts[1]
|
||||
var target: Dictionary = ServerConfig.parse_host_port(String(options["server"]), ServerConfig.DEFAULT_PORT)
|
||||
if target.is_empty():
|
||||
printerr("soak_clients: --server=HOST:PORT is needed")
|
||||
quit(2)
|
||||
return
|
||||
Engine.max_fps = FPS
|
||||
_hash_every = int(options["hash-every"])
|
||||
_tamper = int(options["tamper"])
|
||||
_started_msec = Time.get_ticks_msec()
|
||||
_deadline_msec = _started_msec + int(options["seconds"]) * 1000
|
||||
var first: int = int(options["first"])
|
||||
for index: int in int(options["clients"]):
|
||||
var number: int = first + index
|
||||
var relay: UdpRelay = UdpRelay.new(float(options["latency"]), float(options["jitter"]), float(options["loss"]), int(options["seed"]) * 1000 + number)
|
||||
var error: Error = relay.open(0, String(target["host"]), int(target["port"]))
|
||||
if error != OK:
|
||||
printerr("soak_clients: couldn't open a relay: %s" % error_string(error))
|
||||
quit(2)
|
||||
return
|
||||
var client: Dictionary = {
|
||||
"number": number, "relay": relay, "link": NetClient.new(), "game": ClientGame.new(),
|
||||
"player": SoakPlayer.new(number), "phases": [], "frames_playing": 0,
|
||||
"samples": 0, "skipped": 0, "matches": [], "stats_match": -1,
|
||||
"held": [], "notice": "", "ready_world": null, "ready_seed": 0,
|
||||
}
|
||||
var game: ClientGame = client["game"]
|
||||
var player: SoakPlayer = client["player"]
|
||||
game.sample_input = func(tick: int) -> PlayerCommand:
|
||||
if game.world == null:
|
||||
return PlayerCommand.from_input(tick, Vector2.ZERO, 0.0, 0.0)
|
||||
return player.command(tick, game.prediction.player, game.world)
|
||||
game.on_snapshot = _sample.bind(client)
|
||||
game.make_world = func(match_seed: int) -> Simulation:
|
||||
var ready: Simulation = client["ready_world"]
|
||||
client["ready_world"] = null
|
||||
if ready != null and int(client["ready_seed"]) == match_seed:
|
||||
return ready
|
||||
return Simulation.new(match_seed)
|
||||
game.attach(client["link"])
|
||||
(client["link"] as NetClient).connect_to("127.0.0.1", relay.port(), "Soak%02d" % number)
|
||||
_clients.append(client)
|
||||
|
||||
|
||||
func _process(delta: float) -> bool:
|
||||
if _done:
|
||||
return true
|
||||
var began: int = Time.get_ticks_usec()
|
||||
var open: int = 0
|
||||
var may_make_world: bool = true
|
||||
for client: Dictionary in _clients:
|
||||
var relay: UdpRelay = client["relay"]
|
||||
var link: NetClient = client["link"]
|
||||
var game: ClientGame = client["game"]
|
||||
relay.poll()
|
||||
link.poll()
|
||||
_take_packets(client, link.packets)
|
||||
if may_make_world and not (client["held"] as Array).is_empty():
|
||||
may_make_world = false
|
||||
_make_world(client)
|
||||
game.advance(delta)
|
||||
relay.poll()
|
||||
_follow(client)
|
||||
if link.state != NetClient.CLOSED:
|
||||
open += 1
|
||||
_frame_usec.append(Time.get_ticks_usec() - began)
|
||||
if open == 0 or Time.get_ticks_msec() >= _deadline_msec:
|
||||
_finish()
|
||||
return true
|
||||
return false
|
||||
|
||||
|
||||
## Passes packets to the client's game, holding back a MATCH notice for a new match,
|
||||
## and everything after it, until its world has been made.
|
||||
func _take_packets(client: Dictionary, packets: Array[Dictionary]) -> void:
|
||||
var game: ClientGame = client["game"]
|
||||
var held: Array = client["held"]
|
||||
for packet: Dictionary in packets:
|
||||
if held.is_empty() and not _brings_new_match(client, packet):
|
||||
game.receive(int(packet["channel"]), packet["bytes"])
|
||||
else:
|
||||
held.append(packet)
|
||||
|
||||
|
||||
## True for a MATCH notice of a match (index and seed) the client hasn't been told of.
|
||||
func _brings_new_match(client: Dictionary, packet: Dictionary) -> bool:
|
||||
var bytes: PackedByteArray = packet["bytes"]
|
||||
if int(packet["channel"]) != NetProtocol.CHANNEL_CONTROL or bytes.is_empty() or bytes[0] != NetSnapshot.MSG_MATCH:
|
||||
return false
|
||||
var notice: Dictionary = NetSnapshot.decode_match(bytes)
|
||||
if notice.is_empty():
|
||||
return false
|
||||
var key: String = "%d:%d" % [notice["index"], notice["seed"]]
|
||||
if key == String(client["notice"]):
|
||||
return false
|
||||
client["notice"] = key
|
||||
client["ready_seed"] = int(notice["seed"])
|
||||
return true
|
||||
|
||||
|
||||
## Makes the world for the held notice, then passes on everything held.
|
||||
func _make_world(client: Dictionary) -> void:
|
||||
var game: ClientGame = client["game"]
|
||||
client["ready_world"] = Simulation.new(int(client["ready_seed"]))
|
||||
var held: Array = client["held"]
|
||||
client["held"] = []
|
||||
for packet: Dictionary in held:
|
||||
game.receive(int(packet["channel"]), packet["bytes"])
|
||||
|
||||
|
||||
## Keeps track of a client's phases, and reports each match it played once it's over.
|
||||
func _follow(client: Dictionary) -> void:
|
||||
var game: ClientGame = client["game"]
|
||||
var link: NetClient = client["link"]
|
||||
if link.state == NetClient.CLOSED and not client.has("closed_s"):
|
||||
# Say at once when a connection closes, with what its relay last saw each way.
|
||||
var relay: UdpRelay = client["relay"]
|
||||
var now: int = Time.get_ticks_msec()
|
||||
client["closed_s"] = roundi((now - _started_msec) / 1000.0)
|
||||
print(JSON.stringify({
|
||||
"event": "client_closed", "client": client["number"], "reason": link.close_reason,
|
||||
"at_s": client["closed_s"], "phase": Match.PHASE_NAMES[game.match_phase], "match": game.match_index,
|
||||
"held": (client["held"] as Array).size(), "relay_in_flight": relay.in_flight(),
|
||||
"since_up_ms": now - relay.last_up_msec if relay.last_up_msec >= 0 else -1,
|
||||
"since_down_ms": now - relay.last_down_msec if relay.last_down_msec >= 0 else -1,
|
||||
}))
|
||||
if game.world == null:
|
||||
return
|
||||
var phases: Array = client["phases"]
|
||||
if phases.is_empty() or phases[phases.size() - 1] != game.match_phase:
|
||||
phases.append(game.match_phase)
|
||||
if game.playing():
|
||||
client["frames_playing"] = int(client["frames_playing"]) + 1
|
||||
client["stats_match"] = game.match_index
|
||||
client["last_stats"] = [game.prediction.reconciles, game.prediction.mispredictions, game.prediction.snaps, game.prediction.max_correction]
|
||||
elif int(client["stats_match"]) >= 0:
|
||||
_report_match(client)
|
||||
# Once a match is in play every client here has let go of the last match's world.
|
||||
if game.match_phase == Match.PLAY and not _memory_logged.has(game.match_index):
|
||||
_memory_logged[game.match_index] = true
|
||||
print(JSON.stringify({"event": "memory", "match": game.match_index, "mem_bytes": OS.get_static_memory_usage()}))
|
||||
|
||||
|
||||
func _report_match(client: Dictionary) -> void:
|
||||
var stats: Array = client["last_stats"]
|
||||
var record: Dictionary = {
|
||||
"event": "client_match", "client": client["number"], "match": client["stats_match"],
|
||||
"reconciles": stats[0], "mispredictions": stats[1], "snaps": stats[2], "max_correction": snappedf(stats[3], 0.01),
|
||||
}
|
||||
(client["matches"] as Array).append(record)
|
||||
print(JSON.stringify(record))
|
||||
client["stats_match"] = -1
|
||||
|
||||
|
||||
## After each snapshot: on the server's sampling ticks, the client's world hash, if its
|
||||
## world has caught up to exactly what the snapshot says the server had sent.
|
||||
func _sample(client: Dictionary) -> void:
|
||||
var game: ClientGame = client["game"]
|
||||
var tick: int = game.receiver.sim_tick
|
||||
# Tick 0 lasts the whole lobby and countdown while the world is still being set up,
|
||||
# so it names more than one state; the server doesn't log it either.
|
||||
if _hash_every <= 0 or game.world == null or tick == 0 or tick % _hash_every >= Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
||||
return
|
||||
if game.replica.seq != game.receiver.world_seq or game.replica.pending_count() > 0:
|
||||
client["skipped"] = int(client["skipped"]) + 1
|
||||
return
|
||||
client["samples"] = int(client["samples"]) + 1
|
||||
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
|
||||
_tamper = -1
|
||||
game.world.props.health[0] -= 1
|
||||
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
|
||||
|
||||
|
||||
func _finish() -> void:
|
||||
_done = true
|
||||
var totals: Dictionary = {"up_bytes": 0, "down_bytes": 0, "up_packets": 0, "down_packets": 0, "up_lost": 0, "down_lost": 0}
|
||||
for client: Dictionary in _clients:
|
||||
var relay: UdpRelay = client["relay"]
|
||||
var link: NetClient = client["link"]
|
||||
var game: ClientGame = client["game"]
|
||||
if int(client["stats_match"]) >= 0:
|
||||
_report_match(client)
|
||||
var phase_names: Array[String] = []
|
||||
for phase: int in client["phases"]:
|
||||
phase_names.append(Match.PHASE_NAMES[phase])
|
||||
print(JSON.stringify({
|
||||
"event": "soak_client", "client": client["number"], "name": link.player_name,
|
||||
"joined": link.player_name != "", "close_reason": link.close_reason, "phases": phase_names,
|
||||
"frames_playing": client["frames_playing"], "bad_packets": game.bad_packets,
|
||||
"samples": client["samples"], "skipped": client["skipped"], "matches": (client["matches"] as Array).size(),
|
||||
"build_attempts": (client["player"] as SoakPlayer).build_attempts,
|
||||
"pieces_confirmed": game.replica.confirmed, "pieces_rolled_back": game.replica.rolled_back,
|
||||
"world_gaps": game.replica.gaps,
|
||||
"down_bytes": relay.down_bytes, "up_bytes": relay.up_bytes,
|
||||
}))
|
||||
for key: String in totals:
|
||||
totals[key] = int(totals[key]) + int(relay.get(key))
|
||||
link.leave()
|
||||
relay.close()
|
||||
# The input and snapshot callables hold the client: let go of them, or nothing frees.
|
||||
game.sample_input = Callable()
|
||||
game.on_snapshot = Callable()
|
||||
game.make_world = func(match_seed: int) -> Simulation: return Simulation.new(match_seed)
|
||||
client["ready_world"] = null
|
||||
var sorted: PackedInt64Array = _frame_usec.duplicate()
|
||||
sorted.sort()
|
||||
var summary: Dictionary = {
|
||||
"event": "soak_process", "clients": _clients.size(), "seconds": roundi((Time.get_ticks_msec() - _started_msec) / 1000.0),
|
||||
"frames": sorted.size(), "mem_bytes": OS.get_static_memory_usage(), "errors": _errors.count(),
|
||||
}
|
||||
if not sorted.is_empty():
|
||||
summary["frame_p50_us"] = ServerLoop._percentile(sorted, 0.50)
|
||||
summary["frame_p99_us"] = ServerLoop._percentile(sorted, 0.99)
|
||||
summary.merge(totals)
|
||||
if _errors.count() > 0:
|
||||
summary["first_error"] = _errors.first()
|
||||
print(JSON.stringify(summary))
|
||||
_clients.clear()
|
||||
OS.remove_logger(_errors)
|
||||
quit(1 if _errors.count() > 0 else 0)
|
||||
@@ -0,0 +1 @@
|
||||
uid://gsjp33cla4sm
|
||||
@@ -0,0 +1,227 @@
|
||||
## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the
|
||||
## server ran its matches cleanly and within its tick budget, every client joined and
|
||||
## played every match with no unreadable packets, every client's copy of the world
|
||||
## matched the server's wherever both took a hash, and neither the server nor the
|
||||
## client processes grew in memory from one match to the next.
|
||||
##
|
||||
## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH
|
||||
## --client-logs=PATH[,PATH...] --clients=N --matches=N
|
||||
##
|
||||
## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own
|
||||
## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line.
|
||||
extends SceneTree
|
||||
|
||||
## How much a process's memory may grow from the first match to the last before it
|
||||
## counts as a leak. Each match makes a new world, so some wobble is normal.
|
||||
const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024
|
||||
## Hash comparisons each client must reach per match, so a run that compared nothing
|
||||
## can't pass.
|
||||
const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10
|
||||
|
||||
var _failures: Array[String] = []
|
||||
|
||||
|
||||
func _initialize() -> void:
|
||||
var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"}
|
||||
for arg: String in OS.get_cmdline_user_args():
|
||||
var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1)
|
||||
if parts.size() != 2 or not options.has(parts[0]):
|
||||
printerr("soak_report: unknown argument '%s'" % arg)
|
||||
quit(2)
|
||||
return
|
||||
options[parts[0]] = parts[1]
|
||||
if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty():
|
||||
printerr("soak_report: --server-log and --client-logs are needed")
|
||||
quit(2)
|
||||
return
|
||||
var result: Dictionary = check(
|
||||
_read(String(options["server-log"])),
|
||||
_read_all(String(options["client-logs"]).split(",")),
|
||||
int(options["clients"]), int(options["matches"]))
|
||||
print(JSON.stringify(result))
|
||||
for failure: String in _failures:
|
||||
print("soak FAIL: " + failure)
|
||||
quit(0 if _failures.is_empty() else 1)
|
||||
|
||||
|
||||
## Checks a run from its parsed log lines. Returns the measurements; failures go in
|
||||
## `_failures`.
|
||||
func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary:
|
||||
var result: Dictionary = {"event": "soak"}
|
||||
# The server.
|
||||
var stats: Dictionary = _first(server, "stats")
|
||||
var stop: Dictionary = _first(server, "stop")
|
||||
if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0:
|
||||
_fail("the server didn't stop cleanly: %s" % JSON.stringify(stop))
|
||||
var ended: int = _count(server, "results")
|
||||
result["matches_ended"] = ended
|
||||
if ended < matches:
|
||||
_fail("the server ended %d matches, not %d" % [ended, matches])
|
||||
var budget: int = int(stats.get("budget_us", 0))
|
||||
result["tick_p99_us"] = stats.get("with_net_p99_us", -1)
|
||||
result["tick_max_us"] = stats.get("with_net_max_us", -1)
|
||||
result["sim_p99_us"] = stats.get("p99_us", -1)
|
||||
result["over_budget_ticks"] = stats.get("over_budget", -1)
|
||||
if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget:
|
||||
_fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget])
|
||||
result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server")
|
||||
# The clients.
|
||||
var reports: Array[Dictionary] = _all(clients, "soak_client")
|
||||
result["clients"] = reports.size()
|
||||
if reports.size() != client_count:
|
||||
_fail("%d clients reported, not %d" % [reports.size(), client_count])
|
||||
var down: int = 0
|
||||
var up: int = 0
|
||||
var played: Dictionary = {}
|
||||
for record: Dictionary in _all(clients, "client_match"):
|
||||
played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1
|
||||
for report: Dictionary in reports:
|
||||
var number: int = int(report["client"])
|
||||
down += int(report["down_bytes"])
|
||||
up += int(report["up_bytes"])
|
||||
if not report["joined"]:
|
||||
_fail("client %d never joined" % number)
|
||||
if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN:
|
||||
_fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)])
|
||||
if int(report["bad_packets"]) != 0:
|
||||
_fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]])
|
||||
if int(played.get(number, 0)) < matches:
|
||||
_fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches])
|
||||
var processes: Array[Dictionary] = _all(clients, "soak_process")
|
||||
var seconds: int = 0
|
||||
var mispredictions: int = 0
|
||||
var reconciles: int = 0
|
||||
var snaps: int = 0
|
||||
for record: Dictionary in _all(clients, "client_match"):
|
||||
mispredictions += int(record["mispredictions"])
|
||||
reconciles += int(record["reconciles"])
|
||||
snaps += int(record["snaps"])
|
||||
var lost: int = 0
|
||||
var sent: int = 0
|
||||
for process: Dictionary in processes:
|
||||
seconds = maxi(seconds, int(process["seconds"]))
|
||||
lost += int(process["up_lost"]) + int(process["down_lost"])
|
||||
sent += int(process["up_packets"]) + int(process["down_packets"])
|
||||
if int(process["errors"]) != 0:
|
||||
_fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")])
|
||||
result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0)))
|
||||
result["client_processes"] = processes.size()
|
||||
result["seconds"] = seconds
|
||||
result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001)
|
||||
if seconds > 0 and not reports.is_empty():
|
||||
result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
||||
result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1)
|
||||
result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1)
|
||||
result["reconciles"] = reconciles
|
||||
result["mispredictions"] = mispredictions
|
||||
result["prediction_snaps"] = snaps
|
||||
result["client_mem_growth_bytes"] = _client_growth(clients)
|
||||
# The world, compared hash by hash.
|
||||
var server_hashes: Dictionary = {}
|
||||
for record: Dictionary in _all(server, "world_hash"):
|
||||
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
|
||||
var compared: int = 0
|
||||
var differ: int = 0
|
||||
var per_client: Dictionary = {}
|
||||
for record: Dictionary in _all(clients, "hash"):
|
||||
var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"]))
|
||||
if not server_hashes.has(key):
|
||||
continue
|
||||
compared += 1
|
||||
per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1
|
||||
if server_hashes[key] != String(record["hash"]):
|
||||
differ += 1
|
||||
if differ <= 5:
|
||||
_fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y])
|
||||
result["hashes_compared"] = compared
|
||||
result["hashes_differing"] = differ
|
||||
if differ > 5:
|
||||
_fail("%d world hashes differed in all" % differ)
|
||||
for report: Dictionary in reports:
|
||||
var number: int = int(report["client"])
|
||||
if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches:
|
||||
_fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)])
|
||||
return result
|
||||
|
||||
|
||||
## How much the server's memory grew from the first `phase` entry it logged to the last.
|
||||
func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int:
|
||||
var samples: Array[int] = []
|
||||
for record: Dictionary in _all(lines, event):
|
||||
if String(record.get("to", "")) == phase and record.has("mem_bytes"):
|
||||
samples.append(int(record["mem_bytes"]))
|
||||
return _judge(samples, who)
|
||||
|
||||
|
||||
## The same for each client process, from its "memory" lines.
|
||||
func _client_growth(lines: Array[Dictionary]) -> int:
|
||||
var by_process: Dictionary = {}
|
||||
for record: Dictionary in lines:
|
||||
if String(record.get("event", "")) == "memory":
|
||||
var process: int = int(record.get("_process", 0))
|
||||
if not by_process.has(process):
|
||||
by_process[process] = [] as Array[int]
|
||||
(by_process[process] as Array[int]).append(int(record["mem_bytes"]))
|
||||
var worst: int = 0
|
||||
for process: int in by_process:
|
||||
worst = maxi(worst, _judge(by_process[process], "client process %d" % process))
|
||||
return worst
|
||||
|
||||
|
||||
func _judge(samples: Array[int], who: String) -> int:
|
||||
if samples.size() < 2:
|
||||
_fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()])
|
||||
return 0
|
||||
var growth: int = samples[samples.size() - 1] - samples[0]
|
||||
if growth > LEAK_ALLOWANCE_BYTES:
|
||||
_fail("%s grew by %d bytes from the first match to the last" % [who, growth])
|
||||
return growth
|
||||
|
||||
|
||||
func _fail(message: String) -> void:
|
||||
_failures.append(message)
|
||||
|
||||
|
||||
func _read_all(paths: PackedStringArray) -> Array[Dictionary]:
|
||||
var lines: Array[Dictionary] = []
|
||||
for index: int in paths.size():
|
||||
for record: Dictionary in _read(paths[index]):
|
||||
record["_process"] = index
|
||||
lines.append(record)
|
||||
return lines
|
||||
|
||||
|
||||
## The JSON object lines of a log; anything else in it is skipped.
|
||||
func _read(path: String) -> Array[Dictionary]:
|
||||
var lines: Array[Dictionary] = []
|
||||
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
|
||||
if file == null:
|
||||
_fail("couldn't read %s" % path)
|
||||
return lines
|
||||
while not file.eof_reached():
|
||||
var line: String = file.get_line()
|
||||
if not line.begins_with("{"):
|
||||
continue
|
||||
var parsed: Variant = JSON.parse_string(line)
|
||||
if parsed is Dictionary:
|
||||
lines.append(parsed)
|
||||
return lines
|
||||
|
||||
|
||||
func _first(lines: Array[Dictionary], event: String) -> Dictionary:
|
||||
for record: Dictionary in lines:
|
||||
if String(record.get("event", "")) == event:
|
||||
return record
|
||||
return {}
|
||||
|
||||
|
||||
func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
|
||||
var found: Array[Dictionary] = []
|
||||
for record: Dictionary in lines:
|
||||
if String(record.get("event", "")) == event:
|
||||
found.append(record)
|
||||
return found
|
||||
|
||||
|
||||
func _count(lines: Array[Dictionary], event: String) -> int:
|
||||
return _all(lines, event).size()
|
||||
@@ -0,0 +1 @@
|
||||
uid://byjci1haeqpso
|
||||
Reference in New Issue
Block a user