Add a headless frame-time benchmark and record the baseline
Ashlar checks / tooling (push) Successful in 43s
Ashlar checks / tooling (push) Successful in 43s
docker/ashlar-bench.sh boots the Game Template with fixed video settings, records OpenMW's per-frame profiler trace, and summarizes it with a new stdlib-only script. By default it renders on the NVIDIA GPU through EGL and SDL's offscreen driver, so no window opens. docs/ashlar/BENCHMARK.md holds the method and the first RTX 5060 Ti baseline. Closes #3 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -65,6 +65,14 @@ present. Settings and saves go to `~/.cache/ashlar/play-morrowind/`.
|
||||
MORROWIND_DATA="/path/to/Data Files" docker/ashlar-play.sh
|
||||
```
|
||||
|
||||
To record a frame-time baseline on the NVIDIA GPU without opening a window:
|
||||
|
||||
```bash
|
||||
docker/ashlar-bench.sh
|
||||
```
|
||||
|
||||
See `docs/ashlar/BENCHMARK.md` for the method and the recorded baseline.
|
||||
|
||||
## Git remotes
|
||||
|
||||
- `origin` is `git@gitea.eom.dev:Gaming/ashlar.git`.
|
||||
|
||||
Executable
+111
@@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# Records a frame-time baseline on the asset-free Game Template.
|
||||
#
|
||||
# The engine starts a new game with fixed video settings, the player stands at
|
||||
# the template's start position, and OpenMW's built-in profiler writes every
|
||||
# frame's timings for a fixed duration. The trace and a summary go to
|
||||
# ~/.cache/ashlar/bench/<label>/.
|
||||
#
|
||||
# RENDERER=gpu NVIDIA GPU through EGL with SDL's offscreen driver, so
|
||||
# no window opens (default)
|
||||
# RENDERER=software Xvfb and llvmpipe. Only useful for spotting large CPU
|
||||
# regressions; it says nothing about GPU performance.
|
||||
#
|
||||
# Other settings: LABEL (default: <renderer>-<commit>), SECONDS_TO_RECORD
|
||||
# (default 60), WARMUP_FRAMES (default 300), WIDTH and HEIGHT (default
|
||||
# 1920x1080). Extra arguments go to openmw.
|
||||
# Run docker/ashlar-build.sh first.
|
||||
set -euo pipefail
|
||||
|
||||
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cache="${ASHLAR_CACHE_DIR:-$HOME/.cache/ashlar}"
|
||||
renderer="${RENDERER:-gpu}"
|
||||
record_seconds="${SECONDS_TO_RECORD:-60}"
|
||||
warmup_frames="${WARMUP_FRAMES:-300}"
|
||||
width="${WIDTH:-1920}"
|
||||
height="${HEIGHT:-1080}"
|
||||
memory="${MEMORY:-6g}"
|
||||
image="ashlar-build:trixie"
|
||||
suite="$cache/content/example-suite"
|
||||
commit="$(git -C "$repo" rev-parse --short HEAD)"
|
||||
label="${LABEL:-$renderer-$commit}"
|
||||
out="$cache/bench/$label"
|
||||
|
||||
[ -x "$cache/build/openmw" ] || { echo "Run docker/ashlar-build.sh first." >&2; exit 1; }
|
||||
[ -d "$suite" ] || git clone --depth 1 https://gitlab.com/OpenMW/example-suite.git "$suite"
|
||||
case "$renderer" in gpu|software) ;; *) echo "RENDERER must be gpu or software." >&2; exit 1 ;; esac
|
||||
|
||||
docker build -q -t "$image" -f "$repo/docker/Dockerfile.ashlar" "$repo/docker" >/dev/null
|
||||
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out/home/.config/openmw"
|
||||
cat > "$out/home/.config/openmw/openmw.cfg" <<'EOF'
|
||||
data="/cache/content/example-suite/game_template/data"
|
||||
content=template.omwgame
|
||||
EOF
|
||||
# Fixed settings, so runs are comparable: no frame cap and no vsync.
|
||||
{
|
||||
cat "$suite/settings.cfg"
|
||||
printf '\n[Video]\nresolution x = %s\nresolution y = %s\nwindow mode = 2\n' "$width" "$height"
|
||||
printf 'vsync mode = 0\nframerate limit = 0\n'
|
||||
} > "$out/home/.config/openmw/settings.cfg"
|
||||
|
||||
args=(
|
||||
--rm
|
||||
--name "ashlar-bench-$$"
|
||||
--memory="$memory"
|
||||
--user "$(id -u):$(id -g)"
|
||||
--env HOME="/cache/bench/$label/home"
|
||||
--env OPENMW_OSG_STATS_FILE="/cache/bench/$label/stats"
|
||||
--env OPENMW_OSG_STATS_LIST="times"
|
||||
--volume "$cache:/cache"
|
||||
--workdir /cache/build
|
||||
)
|
||||
if [ "$renderer" = gpu ]; then
|
||||
args+=(
|
||||
--device nvidia.com/gpu=all
|
||||
--env NVIDIA_DRIVER_CAPABILITIES=all
|
||||
--env SDL_VIDEODRIVER=offscreen
|
||||
--env __EGL_VENDOR_LIBRARY_FILENAMES=/usr/share/glvnd/egl_vendor.d/10_nvidia.json
|
||||
)
|
||||
display_setup=":"
|
||||
else
|
||||
args+=(--env DISPLAY=:99 --env LIBGL_ALWAYS_SOFTWARE=1)
|
||||
display_setup="Xvfb :99 -screen 0 ${width}x${height}x24 +extension GLX >/dev/null 2>&1 & sleep 2"
|
||||
fi
|
||||
|
||||
# The engine loads for a few seconds before frames start; those frames are
|
||||
# discarded by WARMUP_FRAMES.
|
||||
docker run "${args[@]}" "$image" bash -uc "
|
||||
$display_setup
|
||||
./openmw --skip-menu --new-game --no-sound $* > /cache/bench/$label/stdout.log 2>&1 &
|
||||
game=\$!
|
||||
sleep '$record_seconds'
|
||||
if ! kill -0 \$game 2>/dev/null; then
|
||||
wait \$game; status=\$?
|
||||
echo \"openmw exited early with status \$status\"
|
||||
tail -40 /cache/bench/$label/stdout.log
|
||||
exit 1
|
||||
fi
|
||||
kill \$game
|
||||
wait \$game 2>/dev/null
|
||||
exit 0
|
||||
"
|
||||
|
||||
grep -m1 -iE 'GL_RENDERER|OpenGL Renderer|Renderer:' "$out/stdout.log" > "$out/renderer.txt" || true
|
||||
{
|
||||
echo "# Ashlar frame-time baseline: $label"
|
||||
echo
|
||||
echo "- Commit: $commit"
|
||||
echo "- Renderer: $renderer ($(cat "$out/renderer.txt" 2>/dev/null || echo unknown))"
|
||||
echo "- Scene: Game Template new game, player idle at the start position"
|
||||
echo "- Resolution: ${width}x${height}, vsync off, no frame cap, sound off"
|
||||
echo "- Recorded: ${record_seconds}s, first $warmup_frames frames skipped"
|
||||
echo "- Note: the profiler itself adds overhead, so compare only runs taken the same way."
|
||||
echo
|
||||
python3 "$repo/scripts/ashlar_bench_summary.py" "$out/stats" \
|
||||
--skip-frames "$warmup_frames" --json "$out/summary.json"
|
||||
} > "$out/summary.md"
|
||||
cat "$out/summary.md"
|
||||
echo
|
||||
echo "Trace: $out/stats"
|
||||
@@ -0,0 +1,17 @@
|
||||
# Ashlar devlog
|
||||
|
||||
Newest entries first. The `ashlar-build` scheduled task and interactive
|
||||
sessions both append here.
|
||||
|
||||
## 2026-11-12
|
||||
|
||||
- #3 performance baseline: added `docker/ashlar-bench.sh` and
|
||||
`scripts/ashlar_bench_summary.py`. The GPU run uses SDL's offscreen driver
|
||||
with NVIDIA EGL, so it is headless. Baseline on the RTX 5060 Ti at 1080p:
|
||||
about 624 mean FPS and 283 1% low over three 60-second runs, within 2%. The
|
||||
method and numbers are in `docs/ashlar/BENCHMARK.md`.
|
||||
- Build and both unit test suites pass (558 tests).
|
||||
- #4 CI is still blocked on the shared runner
|
||||
(DevOps/software-infrastructure#58).
|
||||
- Next: break #5 into sub-issues (active blocking, stamina-driven attacks, hit
|
||||
reactions) and start on blocking.
|
||||
@@ -0,0 +1,49 @@
|
||||
# Performance baseline
|
||||
|
||||
`docker/ashlar-bench.sh` records frame times on the asset-free Game Template so
|
||||
feel and visuals work can be measured against a fixed reference.
|
||||
|
||||
## Method
|
||||
|
||||
- Scene: a new Game Template game with the player idle at the start position
|
||||
(cell Island 0, 0 plus the 8 surrounding cells).
|
||||
- Settings: 1920x1080, vsync off, no frame cap, sound off. Everything else is the
|
||||
example suite's `settings.cfg`.
|
||||
- OpenMW's profiler (`OPENMW_OSG_STATS_LIST=times`) writes every frame's timings
|
||||
to a trace. `scripts/ashlar_bench_summary.py` skips the first 300 frames and
|
||||
reports mean, median, p95, p99 and max per stage.
|
||||
- `RENDERER=gpu` (the default) runs on the NVIDIA GPU through EGL with SDL's
|
||||
offscreen driver, so no window opens. `RENDERER=software` uses Xvfb and
|
||||
llvmpipe; it only catches large CPU regressions.
|
||||
|
||||
Run it after `docker/ashlar-build.sh`:
|
||||
|
||||
```bash
|
||||
docker/ashlar-bench.sh
|
||||
```
|
||||
|
||||
Results go to `~/.cache/ashlar/bench/<label>/` (`summary.md`, `summary.json`,
|
||||
the raw `stats` trace and the engine log). Compare runs only when they were
|
||||
taken the same way, and take at least three, because other work on the host
|
||||
moves the numbers.
|
||||
|
||||
## Baseline: commit 27e35b3ac8, 2026-11-12
|
||||
|
||||
NVIDIA GeForce RTX 5060 Ti (driver 595.104.02), 1920x1080, three 60-second runs:
|
||||
|
||||
| run | frames | mean FPS | 1% low FPS | median frame ms | p99 frame ms |
|
||||
|-----|-------:|---------:|-----------:|----------------:|-------------:|
|
||||
| 1 | 34488 | 636.2 | 283.4 | 1.28 | 3.53 |
|
||||
| 2 | 33341 | 612.4 | 281.7 | 1.33 | 3.55 |
|
||||
| 3 | 34006 | 624.9 | 284.3 | 1.28 | 3.52 |
|
||||
|
||||
The runs agree within about 2%. In run 1, GPU draw time averaged 0.94 ms, cull
|
||||
0.57 ms and update 0.08 ms. The scene is very light, so the
|
||||
numbers mostly show per-frame engine overhead. A heavier, original test scene
|
||||
with many actors and lights is needed before measuring the Visuals milestone.
|
||||
|
||||
Software rendering (llvmpipe, 1280x720, one 60-second run): 49.1 mean FPS,
|
||||
37.0 1% low, 19.9 ms median frame.
|
||||
|
||||
Other scheduled builds were sharing the host during these runs, so treat small
|
||||
differences with caution.
|
||||
Executable
+135
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Summarizes an OpenMW OSG stats trace into frame-time statistics.
|
||||
|
||||
Unlike osg_stats.py this needs only the Python standard library, so it runs on
|
||||
any machine and inside the build container. Times are reported in milliseconds.
|
||||
|
||||
Usage: ashlar_bench_summary.py STATS_FILE [--skip-frames N] [--json OUT]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
# Viewer and camera keys written by osgViewer and OpenMW's profiler, in seconds.
|
||||
KEYS = {
|
||||
"frame": "Frame duration",
|
||||
"event": "Event traversal time taken",
|
||||
"update": "Update traversal time taken",
|
||||
"cull": "Cull traversal time taken",
|
||||
"draw": "Draw traversal time taken",
|
||||
"gpu": "GPU draw time taken",
|
||||
}
|
||||
|
||||
|
||||
def read_frames(stream):
|
||||
"""Yields one dict per frame, merging viewer and camera stats.
|
||||
|
||||
Camera stats are suffixed with " Camera N" when there is more than one
|
||||
camera, matching osg_stats.py. Only the main camera (N=1) is used here.
|
||||
"""
|
||||
frame = {}
|
||||
camera = 0
|
||||
for line in stream:
|
||||
if line.startswith("Stats Viewer"):
|
||||
if frame:
|
||||
yield frame
|
||||
parts = line.split()
|
||||
frame = {"FrameNumber": int(parts[-1])}
|
||||
camera = 0
|
||||
elif line.startswith("Stats Camera"):
|
||||
camera += 1
|
||||
elif line.startswith(" "):
|
||||
parts = line.strip().rsplit(maxsplit=1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
key, value = parts
|
||||
try:
|
||||
number = float(value)
|
||||
except ValueError:
|
||||
continue
|
||||
if camera > 1:
|
||||
continue
|
||||
frame.setdefault(key, number)
|
||||
if frame:
|
||||
yield frame
|
||||
|
||||
|
||||
def percentile(sorted_values, fraction):
|
||||
if not sorted_values:
|
||||
return None
|
||||
index = min(len(sorted_values) - 1, max(0, round(fraction * (len(sorted_values) - 1))))
|
||||
return sorted_values[index]
|
||||
|
||||
|
||||
def summarize(frames, skip_frames):
|
||||
frames = frames[skip_frames:]
|
||||
result = {"frames": len(frames), "metrics": {}}
|
||||
for name, key in KEYS.items():
|
||||
values = sorted(f[key] * 1000.0 for f in frames if key in f)
|
||||
if not values:
|
||||
continue
|
||||
result["metrics"][name] = {
|
||||
"key": key,
|
||||
"samples": len(values),
|
||||
"mean_ms": statistics.fmean(values),
|
||||
"median_ms": statistics.median(values),
|
||||
"p95_ms": percentile(values, 0.95),
|
||||
"p99_ms": percentile(values, 0.99),
|
||||
"max_ms": values[-1],
|
||||
}
|
||||
frame = result["metrics"].get("frame")
|
||||
if frame and frame["mean_ms"] > 0:
|
||||
result["mean_fps"] = 1000.0 / frame["mean_ms"]
|
||||
# The 1% low is the frame rate at the 99th percentile frame time.
|
||||
result["one_percent_low_fps"] = 1000.0 / frame["p99_ms"]
|
||||
return result
|
||||
|
||||
|
||||
def to_markdown(result):
|
||||
lines = [
|
||||
f"Frames analyzed: {result['frames']}",
|
||||
]
|
||||
if "mean_fps" in result:
|
||||
lines.append(f"Mean FPS: {result['mean_fps']:.1f}, 1% low: {result['one_percent_low_fps']:.1f}")
|
||||
lines += [
|
||||
"",
|
||||
"| metric | mean ms | median ms | p95 ms | p99 ms | max ms |",
|
||||
"|--------|--------:|----------:|-------:|-------:|-------:|",
|
||||
]
|
||||
for name, m in result["metrics"].items():
|
||||
lines.append(
|
||||
f"| {name} | {m['mean_ms']:.2f} | {m['median_ms']:.2f} | {m['p95_ms']:.2f} "
|
||||
f"| {m['p99_ms']:.2f} | {m['max_ms']:.2f} |"
|
||||
)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("stats")
|
||||
parser.add_argument("--skip-frames", type=int, default=0, help="warm-up frames to ignore")
|
||||
parser.add_argument("--json", help="also write the summary as JSON to this path")
|
||||
args = parser.parse_args()
|
||||
|
||||
with open(args.stats) as stream:
|
||||
frames = list(read_frames(stream))
|
||||
if len(frames) <= args.skip_frames:
|
||||
print(f"Only {len(frames)} frames in {args.stats}; nothing left after skipping "
|
||||
f"{args.skip_frames}.", file=sys.stderr)
|
||||
return 1
|
||||
result = summarize(frames, args.skip_frames)
|
||||
if "frame" not in result["metrics"]:
|
||||
print(f"No '{KEYS['frame']}' samples in {args.stats}.", file=sys.stderr)
|
||||
return 1
|
||||
if args.json:
|
||||
with open(args.json, "w") as out:
|
||||
json.dump(result, out, indent=2)
|
||||
out.write("\n")
|
||||
print(to_markdown(result))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user