Files
ashlar/scripts/ashlar_bench_summary.py
4a53f5fb50
Ashlar checks / tooling (push) Successful in 43s
Add a headless frame-time benchmark and record the baseline
docker/ashlar-bench.sh boots the Game Template with fixed video settings,
records OpenMW's per-frame profiler trace, and summarizes it with a new
stdlib-only script. By default it renders on the NVIDIA GPU through EGL and
SDL's offscreen driver, so no window opens. docs/ashlar/BENCHMARK.md holds the
method and the first RTX 5060 Ti baseline.

Closes #3

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 13:30:56 -05:00

136 lines
4.4 KiB
Python
Executable File

#!/usr/bin/env python3
"""Summarizes an OpenMW OSG stats trace into frame-time statistics.
Unlike osg_stats.py this needs only the Python standard library, so it runs on
any machine and inside the build container. Times are reported in milliseconds.
Usage: ashlar_bench_summary.py STATS_FILE [--skip-frames N] [--json OUT]
"""
import argparse
import json
import statistics
import sys
# Viewer and camera keys written by osgViewer and OpenMW's profiler, in seconds.
KEYS = {
"frame": "Frame duration",
"event": "Event traversal time taken",
"update": "Update traversal time taken",
"cull": "Cull traversal time taken",
"draw": "Draw traversal time taken",
"gpu": "GPU draw time taken",
}
def read_frames(stream):
"""Yields one dict per frame, merging viewer and camera stats.
Camera stats are suffixed with " Camera N" when there is more than one
camera, matching osg_stats.py. Only the main camera (N=1) is used here.
"""
frame = {}
camera = 0
for line in stream:
if line.startswith("Stats Viewer"):
if frame:
yield frame
parts = line.split()
frame = {"FrameNumber": int(parts[-1])}
camera = 0
elif line.startswith("Stats Camera"):
camera += 1
elif line.startswith(" "):
parts = line.strip().rsplit(maxsplit=1)
if len(parts) != 2:
continue
key, value = parts
try:
number = float(value)
except ValueError:
continue
if camera > 1:
continue
frame.setdefault(key, number)
if frame:
yield frame
def percentile(sorted_values, fraction):
if not sorted_values:
return None
index = min(len(sorted_values) - 1, max(0, round(fraction * (len(sorted_values) - 1))))
return sorted_values[index]
def summarize(frames, skip_frames):
frames = frames[skip_frames:]
result = {"frames": len(frames), "metrics": {}}
for name, key in KEYS.items():
values = sorted(f[key] * 1000.0 for f in frames if key in f)
if not values:
continue
result["metrics"][name] = {
"key": key,
"samples": len(values),
"mean_ms": statistics.fmean(values),
"median_ms": statistics.median(values),
"p95_ms": percentile(values, 0.95),
"p99_ms": percentile(values, 0.99),
"max_ms": values[-1],
}
frame = result["metrics"].get("frame")
if frame and frame["mean_ms"] > 0:
result["mean_fps"] = 1000.0 / frame["mean_ms"]
# The 1% low is the frame rate at the 99th percentile frame time.
result["one_percent_low_fps"] = 1000.0 / frame["p99_ms"]
return result
def to_markdown(result):
lines = [
f"Frames analyzed: {result['frames']}",
]
if "mean_fps" in result:
lines.append(f"Mean FPS: {result['mean_fps']:.1f}, 1% low: {result['one_percent_low_fps']:.1f}")
lines += [
"",
"| metric | mean ms | median ms | p95 ms | p99 ms | max ms |",
"|--------|--------:|----------:|-------:|-------:|-------:|",
]
for name, m in result["metrics"].items():
lines.append(
f"| {name} | {m['mean_ms']:.2f} | {m['median_ms']:.2f} | {m['p95_ms']:.2f} "
f"| {m['p99_ms']:.2f} | {m['max_ms']:.2f} |"
)
return "\n".join(lines)
def main():
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("stats")
parser.add_argument("--skip-frames", type=int, default=0, help="warm-up frames to ignore")
parser.add_argument("--json", help="also write the summary as JSON to this path")
args = parser.parse_args()
with open(args.stats) as stream:
frames = list(read_frames(stream))
if len(frames) <= args.skip_frames:
print(f"Only {len(frames)} frames in {args.stats}; nothing left after skipping "
f"{args.skip_frames}.", file=sys.stderr)
return 1
result = summarize(frames, args.skip_frames)
if "frame" not in result["metrics"]:
print(f"No '{KEYS['frame']}' samples in {args.stats}.", file=sys.stderr)
return 1
if args.json:
with open(args.json, "w") as out:
json.dump(result, out, indent=2)
out.write("\n")
print(to_markdown(result))
return 0
if __name__ == "__main__":
sys.exit(main())