Files
ashlar/docker/ashlar-bench.sh
4a53f5fb50
Ashlar checks / tooling (push) Successful in 43s
Add a headless frame-time benchmark and record the baseline
docker/ashlar-bench.sh boots the Game Template with fixed video settings,
records OpenMW's per-frame profiler trace, and summarizes it with a new
stdlib-only script. By default it renders on the NVIDIA GPU through EGL and
SDL's offscreen driver, so no window opens. docs/ashlar/BENCHMARK.md holds the
method and the first RTX 5060 Ti baseline.

Closes #3

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 13:30:56 -05:00

112 lines
4.2 KiB
Bash
Executable File

#!/usr/bin/env bash
# Records a frame-time baseline on the asset-free Game Template.
#
# The engine starts a new game with fixed video settings, the player stands at
# the template's start position, and OpenMW's built-in profiler writes every
# frame's timings for a fixed duration. The trace and a summary go to
# ~/.cache/ashlar/bench/<label>/.
#
# RENDERER=gpu NVIDIA GPU through EGL with SDL's offscreen driver, so
# no window opens (default)
# RENDERER=software Xvfb and llvmpipe. Only useful for spotting large CPU
# regressions; it says nothing about GPU performance.
#
# Other settings: LABEL (default: <renderer>-<commit>), SECONDS_TO_RECORD
# (default 60), WARMUP_FRAMES (default 300), WIDTH and HEIGHT (default
# 1920x1080). Extra arguments go to openmw.
# Run docker/ashlar-build.sh first.
set -euo pipefail
repo="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cache="${ASHLAR_CACHE_DIR:-$HOME/.cache/ashlar}"
renderer="${RENDERER:-gpu}"
record_seconds="${SECONDS_TO_RECORD:-60}"
warmup_frames="${WARMUP_FRAMES:-300}"
width="${WIDTH:-1920}"
height="${HEIGHT:-1080}"
memory="${MEMORY:-6g}"
image="ashlar-build:trixie"
suite="$cache/content/example-suite"
commit="$(git -C "$repo" rev-parse --short HEAD)"
label="${LABEL:-$renderer-$commit}"
out="$cache/bench/$label"
[ -x "$cache/build/openmw" ] || { echo "Run docker/ashlar-build.sh first." >&2; exit 1; }
[ -d "$suite" ] || git clone --depth 1 https://gitlab.com/OpenMW/example-suite.git "$suite"
case "$renderer" in gpu|software) ;; *) echo "RENDERER must be gpu or software." >&2; exit 1 ;; esac
docker build -q -t "$image" -f "$repo/docker/Dockerfile.ashlar" "$repo/docker" >/dev/null
rm -rf "$out"
mkdir -p "$out/home/.config/openmw"
cat > "$out/home/.config/openmw/openmw.cfg" <<'EOF'
data="/cache/content/example-suite/game_template/data"
content=template.omwgame
EOF
# Fixed settings, so runs are comparable: no frame cap and no vsync.
{
cat "$suite/settings.cfg"
printf '\n[Video]\nresolution x = %s\nresolution y = %s\nwindow mode = 2\n' "$width" "$height"
printf 'vsync mode = 0\nframerate limit = 0\n'
} > "$out/home/.config/openmw/settings.cfg"
args=(
--rm
--name "ashlar-bench-$$"
--memory="$memory"
--user "$(id -u):$(id -g)"
--env HOME="/cache/bench/$label/home"
--env OPENMW_OSG_STATS_FILE="/cache/bench/$label/stats"
--env OPENMW_OSG_STATS_LIST="times"
--volume "$cache:/cache"
--workdir /cache/build
)
if [ "$renderer" = gpu ]; then
args+=(
--device nvidia.com/gpu=all
--env NVIDIA_DRIVER_CAPABILITIES=all
--env SDL_VIDEODRIVER=offscreen
--env __EGL_VENDOR_LIBRARY_FILENAMES=/usr/share/glvnd/egl_vendor.d/10_nvidia.json
)
display_setup=":"
else
args+=(--env DISPLAY=:99 --env LIBGL_ALWAYS_SOFTWARE=1)
display_setup="Xvfb :99 -screen 0 ${width}x${height}x24 +extension GLX >/dev/null 2>&1 & sleep 2"
fi
# The engine loads for a few seconds before frames start; those frames are
# discarded by WARMUP_FRAMES.
docker run "${args[@]}" "$image" bash -uc "
$display_setup
./openmw --skip-menu --new-game --no-sound $* > /cache/bench/$label/stdout.log 2>&1 &
game=\$!
sleep '$record_seconds'
if ! kill -0 \$game 2>/dev/null; then
wait \$game; status=\$?
echo \"openmw exited early with status \$status\"
tail -40 /cache/bench/$label/stdout.log
exit 1
fi
kill \$game
wait \$game 2>/dev/null
exit 0
"
grep -m1 -iE 'GL_RENDERER|OpenGL Renderer|Renderer:' "$out/stdout.log" > "$out/renderer.txt" || true
{
echo "# Ashlar frame-time baseline: $label"
echo
echo "- Commit: $commit"
echo "- Renderer: $renderer ($(cat "$out/renderer.txt" 2>/dev/null || echo unknown))"
echo "- Scene: Game Template new game, player idle at the start position"
echo "- Resolution: ${width}x${height}, vsync off, no frame cap, sound off"
echo "- Recorded: ${record_seconds}s, first $warmup_frames frames skipped"
echo "- Note: the profiler itself adds overhead, so compare only runs taken the same way."
echo
python3 "$repo/scripts/ashlar_bench_summary.py" "$out/stats" \
--skip-frames "$warmup_frames" --json "$out/summary.json"
} > "$out/summary.md"
cat "$out/summary.md"
echo
echo "Trace: $out/stats"