Files
ashlar/scripts/integration_tests.py

293 lines
11 KiB
Python
Executable File

#!/usr/bin/env python3
import argparse
import datetime
import os
import shutil
import subprocess
import sys
import time
from pathlib import Path
def parse_args():
parser = argparse.ArgumentParser(description="OpenMW integration tests.")
parser.add_argument(
"suites",
nargs='*',
type=str,
default=None,
help="suites to run: a directory with suites or subdirectories containing them",
)
parser.add_argument("--omw", type=str, default="openmw", help="path to openmw binary")
parser.add_argument(
"--workdir", type=str, default="integration_tests_output", help="directory for temporary files and logs"
)
parser.add_argument("--verbose", action='store_true', help="print all openmw output")
parser.add_argument(
"--test_filter", type=str, default=None,
help="test cases to run (e.g. 'player.*running')",
)
parser.add_argument(
"--list_tests", action='store_true',
help="print test names instead of running them",
)
parser.add_argument(
"--config", type=str, default=None,
help="extra directory with custom settings.cfg",
)
parser.add_argument(
"--data", action='append', default=[],
help="extra data directories with content files",
)
parser.add_argument(
"--ini", type=str, default=None,
help="INI file to import fallback values from via openmw-iniimporter (found next to --omw)",
)
parser.add_argument(
"--resolution", type=str, default="640x480",
help="window resolution as WIDTHxHEIGHT",
)
return parser.parse_args()
def parse_resolution(value):
width, _, height = value.partition("x")
if not height or not width.isdigit() or not height.isdigit():
sys.exit(f"invalid --resolution {value!r}, expected WIDTHxHEIGHT")
return int(width), int(height)
def lua_string_literal(value):
escaped = value.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
return f'"{escaped}"'
def discover_suites(suite_roots):
all_suites = dict()
for root in suite_roots:
if not root.is_dir():
sys.exit(f"{root} is not a directory")
if (root / "openmw.cfg").is_file():
all_suites[root.name] = root
else:
for entry in sorted(root.glob("test_*")):
if entry.is_dir() and (entry / "openmw.cfg").is_file():
all_suites[entry.name] = entry
if not all_suites:
sys.exit(f"No suites found in: {', '.join(str(v) for v in suite_roots)}")
return all_suites
def write_test_config_module(test_config_dir, test_filter, list_tests):
shutil.rmtree(test_config_dir, ignore_errors=True)
test_config_dir.mkdir(parents=True)
fields = []
if test_filter:
fields.append(f"filter = {lua_string_literal(test_filter)}")
if list_tests:
fields.append("list = true")
with open(test_config_dir / "test_config.lua", "w", encoding="utf-8") as stream:
stream.write(f"return {{ {', '.join(fields)} }}\n")
def parse_test_name(status, line):
return line.split(status)[1].strip().split("\t", maxsplit=1)
def write_generated_config(
config_dir, userdata_dir, data_local_dir, test_config_dir, args, iniimporter_binary, ini_path
):
shutil.rmtree(config_dir, ignore_errors=True)
config_dir.mkdir(parents=True)
base_cfg_path = config_dir / ("openmw.base.cfg" if args.ini else "openmw.cfg")
with open(base_cfg_path, "w", encoding="utf-8") as omw_cfg:
omw_cfg.write(f'user-data="{userdata_dir}"\n')
omw_cfg.write(f'data-local="{data_local_dir}"\n')
omw_cfg.write(f'data="{test_config_dir}"\n')
if args.ini:
subprocess.run(
[
iniimporter_binary,
"--ini", ini_path,
"--cfg", base_cfg_path,
"--output", config_dir / "openmw.cfg",
],
check=True,
)
with open(config_dir / "settings.cfg", "w", encoding="utf-8") as settings_cfg:
width, height = parse_resolution(args.resolution)
settings_cfg.write(
"[Video]\n"
f"resolution x = {width}\n"
f"resolution y = {height}\n"
"framerate limit = 60\n"
"[Game]\n"
"smooth animation transitions = true\n"
"[Lua]\n"
f"memory limit = {1024 * 1024 * 256}\n"
"lua profiler = true\n"
)
def run_test(
suite_name, suite_dir, config_dir, config_source_dir, log_dir, time_str, data_dirs, args, openmw_binary
):
start = time.time()
if not args.list_tests:
print(f'[----------] Running tests from {suite_name}')
command = [openmw_binary, "--replace=config"]
# --config is layered, and settings in a later layer override earlier ones. Pass
# any caller-supplied directory as its own layer rather than concatenating its
# settings.cfg into the generated one: two values for one key in a single file
# is a duplicate and openmw rejects it outright.
#
# The generated directory stays last because openmw writes its runtime files
# into the final layer, and openmw.log is read back from there below.
if config_source_dir is not None:
command += ["--config", config_source_dir]
# A suite outranks the shared directory.
command += ["--config", suite_dir]
command += ["--config", config_dir, "--no-grab"]
for path in data_dirs:
command += ["--data", path]
stdout_lines = list()
fatal_errors = list()
with subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
encoding="utf-8",
env={
"OPENMW_OSG_STATS_FILE": str(log_dir / f"{suite_name}.{time_str}.osg_stats.log"),
"OPENMW_OSG_STATS_LIST": "times",
**os.environ,
},
) as process:
quit_requested = False
running_test_number = None
running_test_name = None
count = 0
failed_tests = list()
test_start = None
for line in process.stdout:
if args.verbose:
sys.stdout.write(line)
else:
stdout_lines.append(line)
if "Quit requested by a Lua script" in line:
quit_requested = True
elif "TEST_FOUND" in line:
_, name = parse_test_name("TEST_FOUND", line)
count += 1
print(name)
elif "TEST_START" in line:
test_start = time.time()
number, name = parse_test_name("TEST_START", line)
running_test_number = int(number)
running_test_name = name
count += 1
print(f"[ RUN ] {running_test_name}")
elif "TEST_OK" in line:
duration = (time.time() - test_start) * 1000
number, name = parse_test_name("TEST_OK", line)
assert running_test_number == int(number)
print(f"[ OK ] {running_test_name} ({duration:.3f} ms)")
elif "TEST_FAILED" in line:
duration = (time.time() - test_start) * 1000
number, name, error = line.split("TEST_FAILED")[1].strip().split("\t", maxsplit=2)
assert running_test_number == int(number)
print(error)
print(f"[ FAILED ] {running_test_name} ({duration:.3f} ms)")
failed_tests.append(running_test_name)
elif "TEST_ERROR" in line:
message = line.split("TEST_ERROR")[1].strip()
fatal_errors.append(f"test framework error: {message}")
process.wait(5)
if not quit_requested:
fatal_errors.append("unexpected termination")
if process.returncode != 0:
fatal_errors.append(f"openmw exited with code {process.returncode}")
if os.path.exists(config_dir / "openmw.log"):
shutil.copyfile(config_dir / "openmw.log", log_dir / f"{suite_name}.{time_str}.log")
if fatal_errors and not args.verbose:
sys.stdout.writelines(stdout_lines)
if not args.list_tests:
total_duration = (time.time() - start) * 1000
print(f'\n[----------] {count} tests from {suite_name} ({total_duration:.3f} ms total)')
print(f"[ PASSED ] {count - len(failed_tests)} tests.")
if fatal_errors:
print(f"[ FAILED ] fatal error: {'; '.join(fatal_errors)}")
if failed_tests:
print(f"[ FAILED ] {len(failed_tests)} tests, listed below:")
for failed_test in failed_tests:
print(f"[ FAILED ] {failed_test}")
return len(failed_tests) == 0 and not fatal_errors, count
def main():
args = parse_args()
openmw_binary = Path(args.omw).resolve()
if not openmw_binary.is_file():
sys.exit(f"{openmw_binary} not found")
iniimporter_binary = None
ini_path = None
if args.ini:
ini_path = Path(args.ini).resolve()
if not ini_path.is_file():
sys.exit(f"{ini_path} not found")
iniimporter_binary = openmw_binary.parent / f"openmw-iniimporter{openmw_binary.suffix}"
if not iniimporter_binary.is_file():
sys.exit(f"{iniimporter_binary} not found")
config_source_dir = Path(args.config).resolve() if args.config is not None else None
data_dirs = [Path(v).resolve() for v in args.data]
work_dir = Path(args.workdir).resolve()
work_dir.mkdir(parents=True, exist_ok=True)
test_config_dir = work_dir / "test_config"
time_str = datetime.datetime.now().strftime("%Y-%m-%d-%H.%M.%S")
suite_roots = [Path(v).resolve() for v in args.suites]
all_suites = discover_suites(suite_roots)
write_test_config_module(test_config_dir=test_config_dir, test_filter=args.test_filter, list_tests=args.list_tests)
status = 0
total_count = 0
for name in sorted(all_suites):
suite_work_dir = work_dir / name
write_generated_config(
config_dir=suite_work_dir,
userdata_dir=suite_work_dir / "userdata",
data_local_dir=suite_work_dir / "data-local",
test_config_dir=test_config_dir,
args=args,
iniimporter_binary=iniimporter_binary,
ini_path=ini_path,
)
result, count = run_test(
suite_name=name,
suite_dir=all_suites[name],
config_dir=suite_work_dir,
config_source_dir=config_source_dir,
log_dir=work_dir,
time_str=time_str,
data_dirs=data_dirs,
args=args,
openmw_binary=openmw_binary,
)
total_count += count
if not result:
status = -1
if args.test_filter and total_count == 0:
print(f"[ WARNING ] no tests matched --test_filter {args.test_filter!r}")
status = -1
exit(status)
if __name__ == "__main__":
main()