Ashlar checks / tooling (push) Successful in 43s
docker/ashlar-bench.sh boots the Game Template with fixed video settings, records OpenMW's per-frame profiler trace, and summarizes it with a new stdlib-only script. By default it renders on the NVIDIA GPU through EGL and SDL's offscreen driver, so no window opens. docs/ashlar/BENCHMARK.md holds the method and the first RTX 5060 Ti baseline. Closes #3 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
136 lines
4.4 KiB
Python
Executable File
136 lines
4.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Summarizes an OpenMW OSG stats trace into frame-time statistics.
|
|
|
|
Unlike osg_stats.py this needs only the Python standard library, so it runs on
|
|
any machine and inside the build container. Times are reported in milliseconds.
|
|
|
|
Usage: ashlar_bench_summary.py STATS_FILE [--skip-frames N] [--json OUT]
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import statistics
|
|
import sys
|
|
|
|
# Viewer and camera keys written by osgViewer and OpenMW's profiler, in seconds.
|
|
KEYS = {
|
|
"frame": "Frame duration",
|
|
"event": "Event traversal time taken",
|
|
"update": "Update traversal time taken",
|
|
"cull": "Cull traversal time taken",
|
|
"draw": "Draw traversal time taken",
|
|
"gpu": "GPU draw time taken",
|
|
}
|
|
|
|
|
|
def read_frames(stream):
|
|
"""Yields one dict per frame, merging viewer and camera stats.
|
|
|
|
Camera stats are suffixed with " Camera N" when there is more than one
|
|
camera, matching osg_stats.py. Only the main camera (N=1) is used here.
|
|
"""
|
|
frame = {}
|
|
camera = 0
|
|
for line in stream:
|
|
if line.startswith("Stats Viewer"):
|
|
if frame:
|
|
yield frame
|
|
parts = line.split()
|
|
frame = {"FrameNumber": int(parts[-1])}
|
|
camera = 0
|
|
elif line.startswith("Stats Camera"):
|
|
camera += 1
|
|
elif line.startswith(" "):
|
|
parts = line.strip().rsplit(maxsplit=1)
|
|
if len(parts) != 2:
|
|
continue
|
|
key, value = parts
|
|
try:
|
|
number = float(value)
|
|
except ValueError:
|
|
continue
|
|
if camera > 1:
|
|
continue
|
|
frame.setdefault(key, number)
|
|
if frame:
|
|
yield frame
|
|
|
|
|
|
def percentile(sorted_values, fraction):
|
|
if not sorted_values:
|
|
return None
|
|
index = min(len(sorted_values) - 1, max(0, round(fraction * (len(sorted_values) - 1))))
|
|
return sorted_values[index]
|
|
|
|
|
|
def summarize(frames, skip_frames):
|
|
frames = frames[skip_frames:]
|
|
result = {"frames": len(frames), "metrics": {}}
|
|
for name, key in KEYS.items():
|
|
values = sorted(f[key] * 1000.0 for f in frames if key in f)
|
|
if not values:
|
|
continue
|
|
result["metrics"][name] = {
|
|
"key": key,
|
|
"samples": len(values),
|
|
"mean_ms": statistics.fmean(values),
|
|
"median_ms": statistics.median(values),
|
|
"p95_ms": percentile(values, 0.95),
|
|
"p99_ms": percentile(values, 0.99),
|
|
"max_ms": values[-1],
|
|
}
|
|
frame = result["metrics"].get("frame")
|
|
if frame and frame["mean_ms"] > 0:
|
|
result["mean_fps"] = 1000.0 / frame["mean_ms"]
|
|
# The 1% low is the frame rate at the 99th percentile frame time.
|
|
result["one_percent_low_fps"] = 1000.0 / frame["p99_ms"]
|
|
return result
|
|
|
|
|
|
def to_markdown(result):
|
|
lines = [
|
|
f"Frames analyzed: {result['frames']}",
|
|
]
|
|
if "mean_fps" in result:
|
|
lines.append(f"Mean FPS: {result['mean_fps']:.1f}, 1% low: {result['one_percent_low_fps']:.1f}")
|
|
lines += [
|
|
"",
|
|
"| metric | mean ms | median ms | p95 ms | p99 ms | max ms |",
|
|
"|--------|--------:|----------:|-------:|-------:|-------:|",
|
|
]
|
|
for name, m in result["metrics"].items():
|
|
lines.append(
|
|
f"| {name} | {m['mean_ms']:.2f} | {m['median_ms']:.2f} | {m['p95_ms']:.2f} "
|
|
f"| {m['p99_ms']:.2f} | {m['max_ms']:.2f} |"
|
|
)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("stats")
|
|
parser.add_argument("--skip-frames", type=int, default=0, help="warm-up frames to ignore")
|
|
parser.add_argument("--json", help="also write the summary as JSON to this path")
|
|
args = parser.parse_args()
|
|
|
|
with open(args.stats) as stream:
|
|
frames = list(read_frames(stream))
|
|
if len(frames) <= args.skip_frames:
|
|
print(f"Only {len(frames)} frames in {args.stats}; nothing left after skipping "
|
|
f"{args.skip_frames}.", file=sys.stderr)
|
|
return 1
|
|
result = summarize(frames, args.skip_frames)
|
|
if "frame" not in result["metrics"]:
|
|
print(f"No '{KEYS['frame']}' samples in {args.stats}.", file=sys.stderr)
|
|
return 1
|
|
if args.json:
|
|
with open(args.json, "w") as out:
|
|
json.dump(result, out, indent=2)
|
|
out.write("\n")
|
|
print(to_markdown(result))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|