## Reads the logs of a soak run (tools/soak.sh) and decides whether it passed: the ## server ran its matches cleanly and within its tick budget, every client joined and ## played every match with no unreadable packets, every client's copy of the world ## matched the server's wherever both took a hash, and neither the server nor the ## client processes grew in memory from one match to the next. ## ## tools/godot.sh --script res://tools/soak_report.gd -- --server-log=PATH ## --client-logs=PATH[,PATH...] --clients=N --matches=N ## ## Prints one {"event":"soak", ...} JSON line of results, then each failure on its own ## line. Exits 0 on a pass, 1 on a failure, 2 on a bad command line. extends SceneTree ## How much a process's memory may grow from the first match to the last before it ## counts as a leak. Each match makes a new world, so some wobble is normal. const LEAK_ALLOWANCE_BYTES: int = 16 * 1024 * 1024 ## Hash comparisons each client must reach per match, so a run that compared nothing ## can't pass. const MIN_COMPARISONS_PER_CLIENT_MATCH: int = 10 var _failures: Array[String] = [] func _initialize() -> void: var options: Dictionary = {"server-log": "", "client-logs": "", "clients": "24", "matches": "2"} for arg: String in OS.get_cmdline_user_args(): var parts: PackedStringArray = arg.trim_prefix("--").split("=", true, 1) if parts.size() != 2 or not options.has(parts[0]): printerr("soak_report: unknown argument '%s'" % arg) quit(2) return options[parts[0]] = parts[1] if String(options["server-log"]).is_empty() or String(options["client-logs"]).is_empty(): printerr("soak_report: --server-log and --client-logs are needed") quit(2) return var result: Dictionary = check( _read(String(options["server-log"])), _read_all(String(options["client-logs"]).split(",")), int(options["clients"]), int(options["matches"])) print(JSON.stringify(result)) for failure: String in _failures: print("soak FAIL: " + failure) quit(0 if _failures.is_empty() else 1) ## Checks a run from its parsed log lines. Returns the measurements; failures go in ## `_failures`. func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: int, matches: int) -> Dictionary: var result: Dictionary = {"event": "soak"} # The server. var stats: Dictionary = _first(server, "stats") var stop: Dictionary = _first(server, "stop") if stop.is_empty() or int(stop.get("exit_code", -1)) != 0 or int(stop.get("errors", -1)) != 0: _fail("the server didn't stop cleanly: %s" % JSON.stringify(stop)) var ended: int = _count(server, "results") result["matches_ended"] = ended if ended < matches: _fail("the server ended %d matches, not %d" % [ended, matches]) var budget: int = int(stats.get("budget_us", 0)) result["tick_p99_us"] = stats.get("with_net_p99_us", -1) result["tick_max_us"] = stats.get("with_net_max_us", -1) result["sim_p99_us"] = stats.get("p99_us", -1) result["over_budget_ticks"] = stats.get("over_budget", -1) if stats.is_empty() or int(stats.get("with_net_p99_us", budget + 1)) > budget: _fail("the server's p99 tick, with the network's share, was %s us, over the %d us budget" % [stats.get("with_net_p99_us", "unknown"), budget]) result["server_mem_growth_bytes"] = _growth(server, "match", "drop", "server") # The clients. var reports: Array[Dictionary] = _all(clients, "soak_client") result["clients"] = reports.size() if reports.size() != client_count: _fail("%d clients reported, not %d" % [reports.size(), client_count]) var down: int = 0 var up: int = 0 var played: Dictionary = {} for record: Dictionary in _all(clients, "client_match"): played[int(record["client"])] = int(played.get(int(record["client"]), 0)) + 1 for report: Dictionary in reports: var number: int = int(report["client"]) down += int(report["down_bytes"]) up += int(report["up_bytes"]) if not report["joined"]: _fail("client %d never joined" % number) if int(report["close_reason"]) != NetProtocol.REASON_SHUTDOWN: _fail("client %d's connection ended early: %s" % [number, NetProtocol.reason_text(int(report["close_reason"]), 0, NetProtocol.VERSION)]) if int(report["bad_packets"]) != 0: _fail("client %d got %d packets it couldn't read" % [number, report["bad_packets"]]) if int(played.get(number, 0)) < matches: _fail("client %d played %d matches, not %d" % [number, played.get(number, 0), matches]) var processes: Array[Dictionary] = _all(clients, "soak_process") var seconds: int = 0 var mispredictions: int = 0 var reconciles: int = 0 var snaps: int = 0 for record: Dictionary in _all(clients, "client_match"): mispredictions += int(record["mispredictions"]) reconciles += int(record["reconciles"]) snaps += int(record["snaps"]) var lost: int = 0 var sent: int = 0 for process: Dictionary in processes: seconds = maxi(seconds, int(process["seconds"])) lost += int(process["up_lost"]) + int(process["down_lost"]) sent += int(process["up_packets"]) + int(process["down_packets"]) if int(process["errors"]) != 0: _fail("a client process logged %d errors, the first: %s" % [process["errors"], process.get("first_error", "")]) result["client_frame_p99_us"] = maxi(int(result.get("client_frame_p99_us", 0)), int(process.get("frame_p99_us", 0))) result["client_processes"] = processes.size() result["seconds"] = seconds result["datagrams_lost_fraction"] = snappedf(float(lost) / maxf(sent, 1.0), 0.0001) if seconds > 0 and not reports.is_empty(): result["down_kbit_s_per_client"] = snappedf(down * 8.0 / 1000.0 / seconds / reports.size(), 0.1) result["up_kbit_s_per_client"] = snappedf(up * 8.0 / 1000.0 / seconds / reports.size(), 0.1) result["server_out_kbit_s"] = snappedf(down * 8.0 / 1000.0 / seconds, 0.1) result["reconciles"] = reconciles result["mispredictions"] = mispredictions result["prediction_snaps"] = snaps result["client_mem_growth_bytes"] = _client_growth(clients) # The world, compared hash by hash. var server_hashes: Dictionary = {} for record: Dictionary in _all(server, "world_hash"): server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"]) var compared: int = 0 var differ: int = 0 var per_client: Dictionary = {} for record: Dictionary in _all(clients, "hash"): var key: Vector2i = Vector2i(int(record["match"]), int(record["sim_tick"])) if not server_hashes.has(key): continue compared += 1 per_client[int(record["client"])] = int(per_client.get(int(record["client"]), 0)) + 1 if server_hashes[key] != String(record["hash"]): differ += 1 if differ <= 5: _fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y]) result["hashes_compared"] = compared result["hashes_differing"] = differ if differ > 5: _fail("%d world hashes differed in all" % differ) for report: Dictionary in reports: var number: int = int(report["client"]) if int(per_client.get(number, 0)) < MIN_COMPARISONS_PER_CLIENT_MATCH * matches: _fail("client %d's world was compared only %d times" % [number, per_client.get(number, 0)]) return result ## How much the server's memory grew from the first `phase` entry it logged to the last. func _growth(lines: Array[Dictionary], event: String, phase: String, who: String) -> int: var samples: Array[int] = [] for record: Dictionary in _all(lines, event): if String(record.get("to", "")) == phase and record.has("mem_bytes"): samples.append(int(record["mem_bytes"])) return _judge(samples, who) ## The same for each client process, from its "memory" lines. func _client_growth(lines: Array[Dictionary]) -> int: var by_process: Dictionary = {} for record: Dictionary in lines: if String(record.get("event", "")) == "memory": var process: int = int(record.get("_process", 0)) if not by_process.has(process): by_process[process] = [] as Array[int] (by_process[process] as Array[int]).append(int(record["mem_bytes"])) var worst: int = 0 for process: int in by_process: worst = maxi(worst, _judge(by_process[process], "client process %d" % process)) return worst func _judge(samples: Array[int], who: String) -> int: if samples.size() < 2: _fail("%s logged its memory at %d match starts, too few to look for a leak" % [who, samples.size()]) return 0 var growth: int = samples[samples.size() - 1] - samples[0] if growth > LEAK_ALLOWANCE_BYTES: _fail("%s grew by %d bytes from the first match to the last" % [who, growth]) return growth func _fail(message: String) -> void: _failures.append(message) func _read_all(paths: PackedStringArray) -> Array[Dictionary]: var lines: Array[Dictionary] = [] for index: int in paths.size(): for record: Dictionary in _read(paths[index]): record["_process"] = index lines.append(record) return lines ## The JSON object lines of a log; anything else in it is skipped. func _read(path: String) -> Array[Dictionary]: var lines: Array[Dictionary] = [] var file: FileAccess = FileAccess.open(path, FileAccess.READ) if file == null: _fail("couldn't read %s" % path) return lines while not file.eof_reached(): var line: String = file.get_line() if not line.begins_with("{"): continue var parsed: Variant = JSON.parse_string(line) if parsed is Dictionary: lines.append(parsed) return lines func _first(lines: Array[Dictionary], event: String) -> Dictionary: for record: Dictionary in lines: if String(record.get("event", "")) == event: return record return {} func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]: var found: Array[Dictionary] = [] for record: Dictionary in lines: if String(record.get("event", "")) == event: found.append(record) return found func _count(lines: Array[Dictionary], event: String) -> int: return _all(lines, event).size()