## Judges a long-run stability run (H4, tools/stability.sh): a dedicated server playing ## matches back to back. Passes when every match ended with a winner inside the time ## limit, the server exited cleanly with no engine or script error, and memory shows no ## growth trend once the first matches have warmed the caches. class_name StabilityReport extends RefCounted ## Matches left out of the memory trend while pools and caches fill. const WARMUP_MATCHES: int = 5 ## The most the server's own (static) memory may grow per match, by least squares over ## the matches after the warmup. const MAX_STATIC_GROWTH_PER_MATCH: float = 256.0 * 1024.0 ## The most the process's resident memory may grow per match, the same way. Looser, ## since the allocator returns memory to the system when it likes. const MAX_RSS_GROWTH_PER_MATCH: float = 1024.0 * 1024.0 ## Least-squares slope of `values` against their index. static func slope(values: Array[float]) -> float: var n: int = values.size() if n < 2: return 0.0 var mean_x: float = (n - 1) / 2.0 var mean_y: float = 0.0 for value: float in values: mean_y += value mean_y /= n var top: float = 0.0 var bottom: float = 0.0 for index: int in n: top += (index - mean_x) * (values[index] - mean_y) bottom += (index - mean_x) * (index - mean_x) return top / bottom ## `log_lines`: the server's stdout and stderr. `rss_lines`: "seconds kilobytes" samples. ## Returns {"matches": [{"seed", "index", "ticks", "seconds", "winners", "timed_out", ## "static_mb", "rss_mb"}], "ended", "errors": [String], "exit_code", "stopped", ## "static_slope", "rss_slope", "peak_rss_mb", "wall_seconds", "problems": [String]}. static func analyse(log_lines: PackedStringArray, rss_lines: PackedStringArray, exit_code: int, stopped: String, expected: int, match_seconds: float) -> Dictionary: var errors: Array[String] = [] # A server without --listen starts its first match before it logs anything. var started: Dictionary = {"tick": 0, "t_ms": 0, "index": 0} var matches: Array[Dictionary] = [] var stop: Dictionary = {} for line: String in log_lines: var parsed: Variant = JSON.parse_string(line) if line.begins_with("{") else null if not parsed is Dictionary: if line.contains("ERROR") or line.contains("SCRIPT ERROR") or line.contains("WARNING: ObjectDB instances leaked"): errors.append(line.strip_edges()) continue var record: Dictionary = parsed var event: String = String(record.get("event", "")) if event == "match" and record.get("type", "") == "match_started": started = record elif event == "match" and record.get("type", "") == "match_ended" and not started.is_empty(): matches.append({ "seed": int(record["match"]), "index": int(started.get("index", matches.size())), "ticks": int(record["tick"]) - int(started["tick"]), "seconds": (float(record["t_ms"]) - float(started["t_ms"])) / 1000.0, "winners": record.get("winners", []), "timed_out": bool(record.get("timed_out", false)), "static_mb": float(record["mem_bytes"]) / 1048576.0, "t_s": float(record["t_ms"]) / 1000.0, }) started = {} elif event == "stop": stop = record # Resident memory at each match's end: the last sample before it. var samples: Array[Vector2] = [] for line: String in rss_lines: var parts: PackedStringArray = line.split(" ", false) if parts.size() == 2 and parts[1].to_int() > 0: samples.append(Vector2(parts[0].to_float(), parts[1].to_float() / 1024.0)) var peak: float = 0.0 for sample: Vector2 in samples: peak = maxf(peak, sample.y) for row: Dictionary in matches: row["rss_mb"] = 0.0 for sample: Vector2 in samples: if sample.x <= row["t_s"] + 1.0: row["rss_mb"] = sample.y var statics: Array[float] = [] var rss: Array[float] = [] for index: int in range(WARMUP_MATCHES, matches.size()): statics.append(float(matches[index]["static_mb"]) * 1048576.0) if matches[index]["rss_mb"] > 0.0: rss.append(float(matches[index]["rss_mb"]) * 1048576.0) var a: Dictionary = { "matches": matches, "ended": matches.size(), "errors": errors, "exit_code": exit_code, "stopped": stopped, "static_slope": slope(statics), "rss_slope": slope(rss), "peak_rss_mb": peak, "wall_seconds": samples[-1].x if not samples.is_empty() else 0.0, "server_errors": int(stop.get("errors", -1)), "problems": [], } var problems: Array = a["problems"] if stopped != "": problems.append("the run was stopped: %s" % stopped) if matches.size() != expected: problems.append("%d of %d matches ended" % [matches.size(), expected]) if exit_code != 0: problems.append("the server exited %d" % exit_code) if stop.is_empty(): problems.append("the server never logged its stop") elif int(stop.get("errors", 0)) != 0: problems.append("the server counted %d errors, first: %s" % [stop["errors"], stop.get("first_error", "")]) if not errors.is_empty(): problems.append("%d error lines, first: %s" % [errors.size(), errors[0]]) for row: Dictionary in matches: if row["winners"].is_empty(): problems.append("match %d (seed %d) ended with no winner" % [row["index"], row["seed"]]) if row["seconds"] > match_seconds: problems.append("match %d (seed %d) took %.0f s, over the %.0f s hang limit" % [row["index"], row["seed"], row["seconds"], match_seconds]) if matches.size() >= WARMUP_MATCHES + 10: if a["static_slope"] > MAX_STATIC_GROWTH_PER_MATCH: problems.append("static memory grows %.0f KiB a match (limit %.0f)" % [a["static_slope"] / 1024.0, MAX_STATIC_GROWTH_PER_MATCH / 1024.0]) if a["rss_slope"] > MAX_RSS_GROWTH_PER_MATCH: problems.append("resident memory grows %.0f KiB a match (limit %.0f)" % [a["rss_slope"] / 1024.0, MAX_RSS_GROWTH_PER_MATCH / 1024.0]) else: problems.append("only %d matches; the memory trend needs at least %d" % [matches.size(), WARMUP_MATCHES + 10]) return a static func format(a: Dictionary) -> String: var lines: PackedStringArray = [] lines.append("Tidewall long-run stability: %d matches back to back on one server, unthrottled" % a["ended"]) lines.append(" wall time %.0f s, exit code %d, server errors %d, error lines %d" % [a["wall_seconds"], a["exit_code"], a["server_errors"], a["errors"].size()]) lines.append(" memory after the first %d matches: static %+.1f KiB a match, resident %+.1f KiB a match; peak resident %.0f MB" % [ WARMUP_MATCHES, a["static_slope"] / 1024.0, a["rss_slope"] / 1024.0, a["peak_rss_mb"]]) lines.append("") lines.append(" # seed ticks wall s winner static MB resident MB") for row: Dictionary in a["matches"]: lines.append(" %-3d %-20d %6d %7.1f %7s %10.1f %12.1f%s" % [row["index"], row["seed"], row["ticks"], row["seconds"], "yes" if not row["winners"].is_empty() else "NO", row["static_mb"], row["rss_mb"], " (tide's clock)" if row["timed_out"] else ""]) lines.append("") if a["problems"].is_empty(): lines.append("stability: ok") else: for problem: String in a["problems"]: lines.append("stability FAIL: %s" % problem) return "\n".join(lines) + "\n"