F4: Headless test harness
Add an in-repo test runner that discovers test/**/test_*.gd and runs under --headless. It uses Godot's Logger API, so engine and script errors during a test fail that test. make test runs the suite, and make test-selfcheck runs deliberately failing fixtures to prove each kind of failure is detected. Closes #4 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -1,10 +1,10 @@
|
|||||||
# Tidewall build entry points. Everything runs headless with the repo-local engine.
|
# Tidewall build entry points. Everything runs headless with the repo-local engine.
|
||||||
GODOT := tools/godot.sh
|
GODOT := tools/godot.sh
|
||||||
|
|
||||||
.PHONY: check import parse binaries verify-engine
|
.PHONY: check import parse test test-selfcheck binaries verify-engine
|
||||||
|
|
||||||
## Everything that must pass before a push.
|
## Everything that must pass before a push.
|
||||||
check: verify-engine binaries parse
|
check: verify-engine binaries parse test test-selfcheck
|
||||||
|
|
||||||
verify-engine:
|
verify-engine:
|
||||||
@$(GODOT) --verify
|
@$(GODOT) --verify
|
||||||
@@ -18,3 +18,10 @@ import:
|
|||||||
|
|
||||||
parse: import
|
parse: import
|
||||||
@$(GODOT) --script res://tools/parse_all.gd
|
@$(GODOT) --script res://tools/parse_all.gd
|
||||||
|
|
||||||
|
test: import
|
||||||
|
@tools/run_tests.sh
|
||||||
|
|
||||||
|
## Proves the test runner detects each kind of failure.
|
||||||
|
test-selfcheck: import
|
||||||
|
@tools/test_selfcheck.sh
|
||||||
|
|||||||
Vendored
Vendored
+5
@@ -0,0 +1,5 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_failing_assertion() -> void:
|
||||||
|
assert_eq(1 + 1, 3, "deliberate fixture failure")
|
||||||
Vendored
@@ -0,0 +1,8 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_engine_error() -> void:
|
||||||
|
var node: Node = Node.new()
|
||||||
|
# The engine reports a parameter error and carries on.
|
||||||
|
node.add_child(null)
|
||||||
|
node.free()
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_declared_error_never_happens() -> void:
|
||||||
|
expect_error("an error that is never raised")
|
||||||
+5
@@ -0,0 +1,5 @@
|
|||||||
|
extends RefCounted
|
||||||
|
|
||||||
|
|
||||||
|
func test_wrong_base_class() -> void:
|
||||||
|
pass
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_does_not_parse() -> void
|
||||||
|
this is not valid GDScript
|
||||||
Vendored
+17
@@ -0,0 +1,17 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_passes() -> void:
|
||||||
|
assert_true(true)
|
||||||
|
assert_eq(1 + 1, 2)
|
||||||
|
assert_near(0.1 + 0.2, 0.3, 0.0001)
|
||||||
|
|
||||||
|
|
||||||
|
func test_expected_engine_error_does_not_fail() -> void:
|
||||||
|
expect_error("deliberate fixture error")
|
||||||
|
push_error("deliberate fixture error")
|
||||||
|
|
||||||
|
|
||||||
|
func test_may_await() -> void:
|
||||||
|
await Engine.get_main_loop().process_frame
|
||||||
|
assert_true(true)
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_null_dereference() -> void:
|
||||||
|
var missing: Node = null
|
||||||
|
missing.get_name()
|
||||||
@@ -0,0 +1,107 @@
|
|||||||
|
## Base class for tests. Files named test_*.gd extend this and define test_* methods.
|
||||||
|
## Test methods may await. Any engine or script error raised during a test fails it,
|
||||||
|
## unless the test declared that error with expect_error().
|
||||||
|
class_name TestCase
|
||||||
|
extends RefCounted
|
||||||
|
|
||||||
|
var _failures: Array[String] = []
|
||||||
|
var _expected_errors: Array[String] = []
|
||||||
|
|
||||||
|
|
||||||
|
## Runs before each test. Override as needed; may await.
|
||||||
|
func before_each() -> void:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
## Runs after each test, even if the test failed. Override as needed; may await.
|
||||||
|
func after_each() -> void:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
## Records a failure. The test keeps running, so one run reports every failure.
|
||||||
|
func fail(message: String) -> void:
|
||||||
|
_failures.append(message)
|
||||||
|
|
||||||
|
|
||||||
|
## Declares that an engine error containing `fragment` is expected in this test.
|
||||||
|
## The test fails if the error does not happen.
|
||||||
|
func expect_error(fragment: String) -> void:
|
||||||
|
_expected_errors.append(fragment)
|
||||||
|
|
||||||
|
|
||||||
|
func assert_true(condition: bool, message: String = "") -> void:
|
||||||
|
if not condition:
|
||||||
|
fail(_describe("expected true", message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_false(condition: bool, message: String = "") -> void:
|
||||||
|
if condition:
|
||||||
|
fail(_describe("expected false", message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_eq(actual: Variant, expected: Variant, message: String = "") -> void:
|
||||||
|
if not _same(actual, expected):
|
||||||
|
fail(_describe("expected %s, got %s" % [_show(expected), _show(actual)], message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_ne(actual: Variant, unexpected: Variant, message: String = "") -> void:
|
||||||
|
if _same(actual, unexpected):
|
||||||
|
fail(_describe("expected a value other than %s" % _show(unexpected), message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_near(actual: float, expected: float, tolerance: float, message: String = "") -> void:
|
||||||
|
if absf(actual - expected) > tolerance:
|
||||||
|
var detail: String = "expected %f within %f, got %f" % [expected, tolerance, actual]
|
||||||
|
fail(_describe(detail, message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_null(value: Variant, message: String = "") -> void:
|
||||||
|
if value != null:
|
||||||
|
fail(_describe("expected null, got %s" % _show(value), message))
|
||||||
|
|
||||||
|
|
||||||
|
func assert_not_null(value: Variant, message: String = "") -> void:
|
||||||
|
if value == null:
|
||||||
|
fail(_describe("expected a value, got null", message))
|
||||||
|
|
||||||
|
|
||||||
|
## Clears per-test state. Called by the runner.
|
||||||
|
func reset_for_test() -> void:
|
||||||
|
_failures = []
|
||||||
|
_expected_errors = []
|
||||||
|
|
||||||
|
|
||||||
|
## Returns this test's failures, including unexpected and missing engine errors.
|
||||||
|
func finish_test(engine_errors: Array[String]) -> Array[String]:
|
||||||
|
var results: Array[String] = _failures.duplicate()
|
||||||
|
var expected: Array[String] = _expected_errors.duplicate()
|
||||||
|
for error: String in engine_errors:
|
||||||
|
var matched: int = -1
|
||||||
|
for index: int in expected.size():
|
||||||
|
if error.contains(expected[index]):
|
||||||
|
matched = index
|
||||||
|
break
|
||||||
|
if matched >= 0:
|
||||||
|
expected.remove_at(matched)
|
||||||
|
else:
|
||||||
|
results.append("engine error: %s" % error)
|
||||||
|
for missing: String in expected:
|
||||||
|
results.append("expected an engine error containing '%s', but none occurred" % missing)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
func _describe(detail: String, message: String) -> String:
|
||||||
|
return detail if message.is_empty() else "%s: %s" % [message, detail]
|
||||||
|
|
||||||
|
|
||||||
|
## Exact comparison that never treats values of different types as equal,
|
||||||
|
## except int and float, which compare by value.
|
||||||
|
func _same(a: Variant, b: Variant) -> bool:
|
||||||
|
var numeric: Array[int] = [TYPE_INT, TYPE_FLOAT]
|
||||||
|
if typeof(a) != typeof(b) and not (typeof(a) in numeric and typeof(b) in numeric):
|
||||||
|
return false
|
||||||
|
return a == b
|
||||||
|
|
||||||
|
|
||||||
|
func _show(value: Variant) -> String:
|
||||||
|
return "'%s'" % value if value is String else str(value)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://b8gr1lo80bhyb
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
## Records engine and script errors so the runner can fail the test that caused them.
|
||||||
|
## Godot may call a Logger from any thread, so all state is guarded by a mutex, and
|
||||||
|
## nothing here prints (printing from a logger callback would recurse).
|
||||||
|
class_name TestLogger
|
||||||
|
extends Logger
|
||||||
|
|
||||||
|
var _mutex: Mutex = Mutex.new()
|
||||||
|
var _errors: Array[String] = []
|
||||||
|
|
||||||
|
|
||||||
|
func _log_error(
|
||||||
|
function: String,
|
||||||
|
file: String,
|
||||||
|
line: int,
|
||||||
|
code: String,
|
||||||
|
rationale: String,
|
||||||
|
_editor_notify: bool,
|
||||||
|
error_type: int,
|
||||||
|
_script_backtrace: Array[ScriptBacktrace],
|
||||||
|
) -> void:
|
||||||
|
if error_type == Logger.ERROR_TYPE_WARNING:
|
||||||
|
return
|
||||||
|
var what: String = rationale if not rationale.is_empty() else code
|
||||||
|
var entry: String = "%s (at %s:%d in %s)" % [what, file, line, function]
|
||||||
|
_mutex.lock()
|
||||||
|
_errors.append(entry)
|
||||||
|
_mutex.unlock()
|
||||||
|
|
||||||
|
|
||||||
|
## Returns every error recorded since the last call, and clears the record.
|
||||||
|
func take_errors() -> Array[String]:
|
||||||
|
_mutex.lock()
|
||||||
|
var taken: Array[String] = _errors
|
||||||
|
_errors = []
|
||||||
|
_mutex.unlock()
|
||||||
|
return taken
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://bpgk2h6e073tn
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
## Headless test runner. Discovers test_*.gd files, runs their test_* methods, and
|
||||||
|
## exits non-zero if anything fails.
|
||||||
|
##
|
||||||
|
## tools/run_tests.sh [--root=res://test/sim] [--filter=substring]
|
||||||
|
extends SceneTree
|
||||||
|
|
||||||
|
const DEFAULT_ROOT: String = "res://test"
|
||||||
|
const FIXTURES_ROOT: String = "res://test/fixtures"
|
||||||
|
|
||||||
|
var _logger: TestLogger = TestLogger.new()
|
||||||
|
var _root: String = DEFAULT_ROOT
|
||||||
|
var _filter: String = ""
|
||||||
|
var _passed: int = 0
|
||||||
|
var _failed: int = 0
|
||||||
|
var _reports: Array[String] = []
|
||||||
|
|
||||||
|
|
||||||
|
func _initialize() -> void:
|
||||||
|
OS.add_logger(_logger)
|
||||||
|
_parse_args()
|
||||||
|
_run.call_deferred()
|
||||||
|
|
||||||
|
|
||||||
|
func _parse_args() -> void:
|
||||||
|
for arg: String in OS.get_cmdline_user_args():
|
||||||
|
if arg.begins_with("--root="):
|
||||||
|
_root = arg.trim_prefix("--root=").trim_suffix("/")
|
||||||
|
elif arg.begins_with("--filter="):
|
||||||
|
_filter = arg.trim_prefix("--filter=")
|
||||||
|
else:
|
||||||
|
push_error("Unknown test runner argument: %s" % arg)
|
||||||
|
|
||||||
|
|
||||||
|
func _run() -> void:
|
||||||
|
var files: Array[String] = []
|
||||||
|
_discover(_root, files)
|
||||||
|
files.sort()
|
||||||
|
# A runner that finds nothing must not look like a passing suite.
|
||||||
|
var ran_nothing: bool = files.is_empty()
|
||||||
|
for path: String in files:
|
||||||
|
await _run_file(path)
|
||||||
|
if _passed + _failed == 0:
|
||||||
|
ran_nothing = true
|
||||||
|
_finish(ran_nothing)
|
||||||
|
|
||||||
|
|
||||||
|
func _discover(dir_path: String, out: Array[String]) -> void:
|
||||||
|
# Fixtures contain deliberate failures. They only run when targeted directly.
|
||||||
|
var skipping_fixtures: bool = dir_path == FIXTURES_ROOT and not _root.begins_with(FIXTURES_ROOT)
|
||||||
|
if skipping_fixtures:
|
||||||
|
return
|
||||||
|
for name: String in DirAccess.get_files_at(dir_path):
|
||||||
|
if name.begins_with("test_") and name.ends_with(".gd"):
|
||||||
|
out.append(dir_path.path_join(name))
|
||||||
|
for name: String in DirAccess.get_directories_at(dir_path):
|
||||||
|
if not name.begins_with("."):
|
||||||
|
_discover(dir_path.path_join(name), out)
|
||||||
|
|
||||||
|
|
||||||
|
func _run_file(path: String) -> void:
|
||||||
|
_logger.take_errors()
|
||||||
|
var script: GDScript = load(path) as GDScript
|
||||||
|
var load_errors: Array[String] = _logger.take_errors()
|
||||||
|
if script == null or not script.can_instantiate():
|
||||||
|
_record_load_failure(path, "could not compile the script", load_errors)
|
||||||
|
return
|
||||||
|
|
||||||
|
var instance: TestCase = script.new() as TestCase
|
||||||
|
if instance == null:
|
||||||
|
_record_load_failure(path, "script does not extend TestCase", _logger.take_errors())
|
||||||
|
return
|
||||||
|
|
||||||
|
var names: Array[String] = _test_names(script)
|
||||||
|
for test_name: String in names:
|
||||||
|
if _filter.is_empty() or (path + ":" + test_name).contains(_filter):
|
||||||
|
await _run_test(instance, path, test_name)
|
||||||
|
|
||||||
|
|
||||||
|
func _test_names(script: GDScript) -> Array[String]:
|
||||||
|
var names: Array[String] = []
|
||||||
|
for method: Dictionary in script.get_script_method_list():
|
||||||
|
var method_name: String = method["name"]
|
||||||
|
if method_name.begins_with("test_") and method_name not in names:
|
||||||
|
names.append(method_name)
|
||||||
|
names.sort()
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
func _run_test(instance: TestCase, path: String, test_name: String) -> void:
|
||||||
|
_logger.take_errors()
|
||||||
|
instance.reset_for_test()
|
||||||
|
await instance.before_each()
|
||||||
|
await instance.call(test_name)
|
||||||
|
await instance.after_each()
|
||||||
|
# Let deferred calls raise their errors while this test is still current.
|
||||||
|
await process_frame
|
||||||
|
_record(path, test_name, instance.finish_test(_logger.take_errors()))
|
||||||
|
|
||||||
|
|
||||||
|
func _record_load_failure(path: String, reason: String, errors: Array[String]) -> void:
|
||||||
|
var failures: Array[String] = [reason]
|
||||||
|
failures.append_array(errors)
|
||||||
|
_record(path, "(load)", failures)
|
||||||
|
|
||||||
|
|
||||||
|
func _record(path: String, test_name: String, failures: Array[String]) -> void:
|
||||||
|
var label: String = "%s::%s" % [path.trim_prefix("res://"), test_name]
|
||||||
|
if failures.is_empty():
|
||||||
|
_passed += 1
|
||||||
|
print("PASS %s" % label)
|
||||||
|
return
|
||||||
|
_failed += 1
|
||||||
|
print("FAIL %s" % label)
|
||||||
|
for failure: String in failures:
|
||||||
|
print(" - %s" % failure)
|
||||||
|
_reports.append("%s: %s" % [label, failure])
|
||||||
|
|
||||||
|
|
||||||
|
func _finish(ran_nothing: bool) -> void:
|
||||||
|
print("")
|
||||||
|
for report: String in _reports:
|
||||||
|
print("FAILED: %s" % report)
|
||||||
|
if ran_nothing:
|
||||||
|
print("No tests ran under %s. Treating this as a failure." % _root)
|
||||||
|
print("Tests: %d passed, %d failed." % [_passed, _failed])
|
||||||
|
quit(1 if _failed > 0 or ran_nothing else 0)
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://cr7xjx1o67nl0
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://paoe2bxdshd7
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
uid://btff4b5o7gimu
|
||||||
@@ -0,0 +1,12 @@
|
|||||||
|
extends TestCase
|
||||||
|
|
||||||
|
|
||||||
|
func test_engine_is_the_pinned_version() -> void:
|
||||||
|
var info: Dictionary = Engine.get_version_info()
|
||||||
|
assert_eq(info["major"], 4)
|
||||||
|
assert_eq(info["minor"], 7)
|
||||||
|
assert_eq(info["patch"], 2)
|
||||||
|
|
||||||
|
|
||||||
|
func test_simulation_runs_at_thirty_hertz() -> void:
|
||||||
|
assert_eq(ProjectSettings.get_setting("physics/common/physics_ticks_per_second"), 30)
|
||||||
Executable
+8
@@ -0,0 +1,8 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Runs the headless test suite. Extra arguments are passed to the runner, for
|
||||||
|
# example: tools/run_tests.sh --root=res://test/sim
|
||||||
|
set -uo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
# A hang in a test must fail the run, not block it forever.
|
||||||
|
exec timeout --signal=KILL "${TIDEWALL_TEST_TIMEOUT:-900}" \
|
||||||
|
tools/godot.sh --script res://test/framework/runner.gd -- "$@"
|
||||||
Executable
+34
@@ -0,0 +1,34 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Proves the test runner detects each kind of failure, using deliberate fixtures.
|
||||||
|
set -uo pipefail
|
||||||
|
cd "$(dirname "${BASH_SOURCE[0]}")/.."
|
||||||
|
|
||||||
|
status=0
|
||||||
|
# check <fixture> <expected exit> <text that must appear in the output>
|
||||||
|
check() {
|
||||||
|
local fixture="$1" expected="$2" needle="$3" output code
|
||||||
|
output="$(tools/run_tests.sh "--root=res://test/fixtures/$fixture" 2>&1)"
|
||||||
|
code=$?
|
||||||
|
if [[ "$code" -ne "$expected" ]]; then
|
||||||
|
echo "selfcheck FAIL $fixture: exit $code, expected $expected"
|
||||||
|
echo "$output" | tail -n 20
|
||||||
|
status=1
|
||||||
|
elif ! grep -qF -- "$needle" <<<"$output"; then
|
||||||
|
echo "selfcheck FAIL $fixture: output lacks '$needle'"
|
||||||
|
echo "$output" | tail -n 20
|
||||||
|
status=1
|
||||||
|
else
|
||||||
|
echo "selfcheck ok $fixture (exit $code)"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check passes 0 "Tests: 3 passed, 0 failed."
|
||||||
|
check assert 1 "deliberate fixture failure: expected 3, got 2"
|
||||||
|
check script_error 1 "engine error:"
|
||||||
|
check engine_error 1 "engine error:"
|
||||||
|
check parse_error 1 "could not compile the script"
|
||||||
|
check not_a_test 1 "does not extend TestCase"
|
||||||
|
check missing_error 1 "expected an engine error containing"
|
||||||
|
check empty 1 "No tests ran"
|
||||||
|
|
||||||
|
exit "$status"
|
||||||
Reference in New Issue
Block a user