diff --git a/Makefile b/Makefile index a33e0b6..79fec78 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ # Tidewall build entry points. Everything runs headless with the repo-local engine. GODOT := tools/godot.sh -.PHONY: check import parse binaries verify-engine +.PHONY: check import parse test test-selfcheck binaries verify-engine ## Everything that must pass before a push. -check: verify-engine binaries parse +check: verify-engine binaries parse test test-selfcheck verify-engine: @$(GODOT) --verify @@ -18,3 +18,10 @@ import: parse: import @$(GODOT) --script res://tools/parse_all.gd + +test: import + @tools/run_tests.sh + +## Proves the test runner detects each kind of failure. +test-selfcheck: import + @tools/test_selfcheck.sh diff --git a/test/fixtures/.gdignore b/test/fixtures/.gdignore new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/assert/test_assert.gd b/test/fixtures/assert/test_assert.gd new file mode 100644 index 0000000..f5d9c28 --- /dev/null +++ b/test/fixtures/assert/test_assert.gd @@ -0,0 +1,5 @@ +extends TestCase + + +func test_failing_assertion() -> void: + assert_eq(1 + 1, 3, "deliberate fixture failure") diff --git a/test/fixtures/empty/.gitkeep b/test/fixtures/empty/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/engine_error/test_engine_error.gd b/test/fixtures/engine_error/test_engine_error.gd new file mode 100644 index 0000000..fdd7c20 --- /dev/null +++ b/test/fixtures/engine_error/test_engine_error.gd @@ -0,0 +1,8 @@ +extends TestCase + + +func test_engine_error() -> void: + var node: Node = Node.new() + # The engine reports a parameter error and carries on. + node.add_child(null) + node.free() diff --git a/test/fixtures/missing_error/test_missing_error.gd b/test/fixtures/missing_error/test_missing_error.gd new file mode 100644 index 0000000..a2c9123 --- /dev/null +++ b/test/fixtures/missing_error/test_missing_error.gd @@ -0,0 +1,5 @@ +extends TestCase + + +func test_declared_error_never_happens() -> void: + expect_error("an error that is never raised") diff --git a/test/fixtures/not_a_test/test_not_a_test.gd b/test/fixtures/not_a_test/test_not_a_test.gd new file mode 100644 index 0000000..a647ff3 --- /dev/null +++ b/test/fixtures/not_a_test/test_not_a_test.gd @@ -0,0 +1,5 @@ +extends RefCounted + + +func test_wrong_base_class() -> void: + pass diff --git a/test/fixtures/parse_error/test_parse_error.gd b/test/fixtures/parse_error/test_parse_error.gd new file mode 100644 index 0000000..61ac78f --- /dev/null +++ b/test/fixtures/parse_error/test_parse_error.gd @@ -0,0 +1,5 @@ +extends TestCase + + +func test_does_not_parse() -> void + this is not valid GDScript diff --git a/test/fixtures/passes/test_passes.gd b/test/fixtures/passes/test_passes.gd new file mode 100644 index 0000000..ed14d3c --- /dev/null +++ b/test/fixtures/passes/test_passes.gd @@ -0,0 +1,17 @@ +extends TestCase + + +func test_passes() -> void: + assert_true(true) + assert_eq(1 + 1, 2) + assert_near(0.1 + 0.2, 0.3, 0.0001) + + +func test_expected_engine_error_does_not_fail() -> void: + expect_error("deliberate fixture error") + push_error("deliberate fixture error") + + +func test_may_await() -> void: + await Engine.get_main_loop().process_frame + assert_true(true) diff --git a/test/fixtures/script_error/test_script_error.gd b/test/fixtures/script_error/test_script_error.gd new file mode 100644 index 0000000..856b80d --- /dev/null +++ b/test/fixtures/script_error/test_script_error.gd @@ -0,0 +1,6 @@ +extends TestCase + + +func test_null_dereference() -> void: + var missing: Node = null + missing.get_name() diff --git a/test/framework/case.gd b/test/framework/case.gd new file mode 100644 index 0000000..b916748 --- /dev/null +++ b/test/framework/case.gd @@ -0,0 +1,107 @@ +## Base class for tests. Files named test_*.gd extend this and define test_* methods. +## Test methods may await. Any engine or script error raised during a test fails it, +## unless the test declared that error with expect_error(). +class_name TestCase +extends RefCounted + +var _failures: Array[String] = [] +var _expected_errors: Array[String] = [] + + +## Runs before each test. Override as needed; may await. +func before_each() -> void: + pass + + +## Runs after each test, even if the test failed. Override as needed; may await. +func after_each() -> void: + pass + + +## Records a failure. The test keeps running, so one run reports every failure. +func fail(message: String) -> void: + _failures.append(message) + + +## Declares that an engine error containing `fragment` is expected in this test. +## The test fails if the error does not happen. +func expect_error(fragment: String) -> void: + _expected_errors.append(fragment) + + +func assert_true(condition: bool, message: String = "") -> void: + if not condition: + fail(_describe("expected true", message)) + + +func assert_false(condition: bool, message: String = "") -> void: + if condition: + fail(_describe("expected false", message)) + + +func assert_eq(actual: Variant, expected: Variant, message: String = "") -> void: + if not _same(actual, expected): + fail(_describe("expected %s, got %s" % [_show(expected), _show(actual)], message)) + + +func assert_ne(actual: Variant, unexpected: Variant, message: String = "") -> void: + if _same(actual, unexpected): + fail(_describe("expected a value other than %s" % _show(unexpected), message)) + + +func assert_near(actual: float, expected: float, tolerance: float, message: String = "") -> void: + if absf(actual - expected) > tolerance: + var detail: String = "expected %f within %f, got %f" % [expected, tolerance, actual] + fail(_describe(detail, message)) + + +func assert_null(value: Variant, message: String = "") -> void: + if value != null: + fail(_describe("expected null, got %s" % _show(value), message)) + + +func assert_not_null(value: Variant, message: String = "") -> void: + if value == null: + fail(_describe("expected a value, got null", message)) + + +## Clears per-test state. Called by the runner. +func reset_for_test() -> void: + _failures = [] + _expected_errors = [] + + +## Returns this test's failures, including unexpected and missing engine errors. +func finish_test(engine_errors: Array[String]) -> Array[String]: + var results: Array[String] = _failures.duplicate() + var expected: Array[String] = _expected_errors.duplicate() + for error: String in engine_errors: + var matched: int = -1 + for index: int in expected.size(): + if error.contains(expected[index]): + matched = index + break + if matched >= 0: + expected.remove_at(matched) + else: + results.append("engine error: %s" % error) + for missing: String in expected: + results.append("expected an engine error containing '%s', but none occurred" % missing) + return results + + +func _describe(detail: String, message: String) -> String: + return detail if message.is_empty() else "%s: %s" % [message, detail] + + +## Exact comparison that never treats values of different types as equal, +## except int and float, which compare by value. +func _same(a: Variant, b: Variant) -> bool: + var numeric: Array[int] = [TYPE_INT, TYPE_FLOAT] + if typeof(a) != typeof(b) and not (typeof(a) in numeric and typeof(b) in numeric): + return false + return a == b + + +func _show(value: Variant) -> String: + return "'%s'" % value if value is String else str(value) diff --git a/test/framework/case.gd.uid b/test/framework/case.gd.uid new file mode 100644 index 0000000..c04a3d4 --- /dev/null +++ b/test/framework/case.gd.uid @@ -0,0 +1 @@ +uid://b8gr1lo80bhyb diff --git a/test/framework/error_log.gd b/test/framework/error_log.gd new file mode 100644 index 0000000..f4b0468 --- /dev/null +++ b/test/framework/error_log.gd @@ -0,0 +1,36 @@ +## Records engine and script errors so the runner can fail the test that caused them. +## Godot may call a Logger from any thread, so all state is guarded by a mutex, and +## nothing here prints (printing from a logger callback would recurse). +class_name TestLogger +extends Logger + +var _mutex: Mutex = Mutex.new() +var _errors: Array[String] = [] + + +func _log_error( + function: String, + file: String, + line: int, + code: String, + rationale: String, + _editor_notify: bool, + error_type: int, + _script_backtrace: Array[ScriptBacktrace], +) -> void: + if error_type == Logger.ERROR_TYPE_WARNING: + return + var what: String = rationale if not rationale.is_empty() else code + var entry: String = "%s (at %s:%d in %s)" % [what, file, line, function] + _mutex.lock() + _errors.append(entry) + _mutex.unlock() + + +## Returns every error recorded since the last call, and clears the record. +func take_errors() -> Array[String]: + _mutex.lock() + var taken: Array[String] = _errors + _errors = [] + _mutex.unlock() + return taken diff --git a/test/framework/error_log.gd.uid b/test/framework/error_log.gd.uid new file mode 100644 index 0000000..26b85d7 --- /dev/null +++ b/test/framework/error_log.gd.uid @@ -0,0 +1 @@ +uid://bpgk2h6e073tn diff --git a/test/framework/runner.gd b/test/framework/runner.gd new file mode 100644 index 0000000..a63fbca --- /dev/null +++ b/test/framework/runner.gd @@ -0,0 +1,126 @@ +## Headless test runner. Discovers test_*.gd files, runs their test_* methods, and +## exits non-zero if anything fails. +## +## tools/run_tests.sh [--root=res://test/sim] [--filter=substring] +extends SceneTree + +const DEFAULT_ROOT: String = "res://test" +const FIXTURES_ROOT: String = "res://test/fixtures" + +var _logger: TestLogger = TestLogger.new() +var _root: String = DEFAULT_ROOT +var _filter: String = "" +var _passed: int = 0 +var _failed: int = 0 +var _reports: Array[String] = [] + + +func _initialize() -> void: + OS.add_logger(_logger) + _parse_args() + _run.call_deferred() + + +func _parse_args() -> void: + for arg: String in OS.get_cmdline_user_args(): + if arg.begins_with("--root="): + _root = arg.trim_prefix("--root=").trim_suffix("/") + elif arg.begins_with("--filter="): + _filter = arg.trim_prefix("--filter=") + else: + push_error("Unknown test runner argument: %s" % arg) + + +func _run() -> void: + var files: Array[String] = [] + _discover(_root, files) + files.sort() + # A runner that finds nothing must not look like a passing suite. + var ran_nothing: bool = files.is_empty() + for path: String in files: + await _run_file(path) + if _passed + _failed == 0: + ran_nothing = true + _finish(ran_nothing) + + +func _discover(dir_path: String, out: Array[String]) -> void: + # Fixtures contain deliberate failures. They only run when targeted directly. + var skipping_fixtures: bool = dir_path == FIXTURES_ROOT and not _root.begins_with(FIXTURES_ROOT) + if skipping_fixtures: + return + for name: String in DirAccess.get_files_at(dir_path): + if name.begins_with("test_") and name.ends_with(".gd"): + out.append(dir_path.path_join(name)) + for name: String in DirAccess.get_directories_at(dir_path): + if not name.begins_with("."): + _discover(dir_path.path_join(name), out) + + +func _run_file(path: String) -> void: + _logger.take_errors() + var script: GDScript = load(path) as GDScript + var load_errors: Array[String] = _logger.take_errors() + if script == null or not script.can_instantiate(): + _record_load_failure(path, "could not compile the script", load_errors) + return + + var instance: TestCase = script.new() as TestCase + if instance == null: + _record_load_failure(path, "script does not extend TestCase", _logger.take_errors()) + return + + var names: Array[String] = _test_names(script) + for test_name: String in names: + if _filter.is_empty() or (path + ":" + test_name).contains(_filter): + await _run_test(instance, path, test_name) + + +func _test_names(script: GDScript) -> Array[String]: + var names: Array[String] = [] + for method: Dictionary in script.get_script_method_list(): + var method_name: String = method["name"] + if method_name.begins_with("test_") and method_name not in names: + names.append(method_name) + names.sort() + return names + + +func _run_test(instance: TestCase, path: String, test_name: String) -> void: + _logger.take_errors() + instance.reset_for_test() + await instance.before_each() + await instance.call(test_name) + await instance.after_each() + # Let deferred calls raise their errors while this test is still current. + await process_frame + _record(path, test_name, instance.finish_test(_logger.take_errors())) + + +func _record_load_failure(path: String, reason: String, errors: Array[String]) -> void: + var failures: Array[String] = [reason] + failures.append_array(errors) + _record(path, "(load)", failures) + + +func _record(path: String, test_name: String, failures: Array[String]) -> void: + var label: String = "%s::%s" % [path.trim_prefix("res://"), test_name] + if failures.is_empty(): + _passed += 1 + print("PASS %s" % label) + return + _failed += 1 + print("FAIL %s" % label) + for failure: String in failures: + print(" - %s" % failure) + _reports.append("%s: %s" % [label, failure]) + + +func _finish(ran_nothing: bool) -> void: + print("") + for report: String in _reports: + print("FAILED: %s" % report) + if ran_nothing: + print("No tests ran under %s. Treating this as a failure." % _root) + print("Tests: %d passed, %d failed." % [_passed, _failed]) + quit(1 if _failed > 0 or ran_nothing else 0) diff --git a/test/framework/runner.gd.uid b/test/framework/runner.gd.uid new file mode 100644 index 0000000..5a63a28 --- /dev/null +++ b/test/framework/runner.gd.uid @@ -0,0 +1 @@ +uid://cr7xjx1o67nl0 diff --git a/test/framework/test_case.gd.uid b/test/framework/test_case.gd.uid new file mode 100644 index 0000000..2c5087e --- /dev/null +++ b/test/framework/test_case.gd.uid @@ -0,0 +1 @@ +uid://paoe2bxdshd7 diff --git a/test/framework/test_logger.gd.uid b/test/framework/test_logger.gd.uid new file mode 100644 index 0000000..fd00780 --- /dev/null +++ b/test/framework/test_logger.gd.uid @@ -0,0 +1 @@ +uid://btff4b5o7gimu diff --git a/test/project/test_layout.gd b/test/project/test_layout.gd new file mode 100644 index 0000000..d38b0f9 --- /dev/null +++ b/test/project/test_layout.gd @@ -0,0 +1,12 @@ +extends TestCase + + +func test_engine_is_the_pinned_version() -> void: + var info: Dictionary = Engine.get_version_info() + assert_eq(info["major"], 4) + assert_eq(info["minor"], 7) + assert_eq(info["patch"], 2) + + +func test_simulation_runs_at_thirty_hertz() -> void: + assert_eq(ProjectSettings.get_setting("physics/common/physics_ticks_per_second"), 30) diff --git a/tools/run_tests.sh b/tools/run_tests.sh new file mode 100755 index 0000000..2881a55 --- /dev/null +++ b/tools/run_tests.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Runs the headless test suite. Extra arguments are passed to the runner, for +# example: tools/run_tests.sh --root=res://test/sim +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +# A hang in a test must fail the run, not block it forever. +exec timeout --signal=KILL "${TIDEWALL_TEST_TIMEOUT:-900}" \ + tools/godot.sh --script res://test/framework/runner.gd -- "$@" diff --git a/tools/test_selfcheck.sh b/tools/test_selfcheck.sh new file mode 100755 index 0000000..207c490 --- /dev/null +++ b/tools/test_selfcheck.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Proves the test runner detects each kind of failure, using deliberate fixtures. +set -uo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +status=0 +# check +check() { + local fixture="$1" expected="$2" needle="$3" output code + output="$(tools/run_tests.sh "--root=res://test/fixtures/$fixture" 2>&1)" + code=$? + if [[ "$code" -ne "$expected" ]]; then + echo "selfcheck FAIL $fixture: exit $code, expected $expected" + echo "$output" | tail -n 20 + status=1 + elif ! grep -qF -- "$needle" <<<"$output"; then + echo "selfcheck FAIL $fixture: output lacks '$needle'" + echo "$output" | tail -n 20 + status=1 + else + echo "selfcheck ok $fixture (exit $code)" + fi +} + +check passes 0 "Tests: 3 passed, 0 failed." +check assert 1 "deliberate fixture failure: expected 3, got 2" +check script_error 1 "engine error:" +check engine_error 1 "engine error:" +check parse_error 1 "could not compile the script" +check not_a_test 1 "does not extend TestCase" +check missing_error 1 "expected an engine error containing" +check empty 1 "No tests ran" + +exit "$status"