N7: Dedicated server packaging, health endpoint and graceful shutdown
- packaging/server/tidewall-server: a wrapper that turns SIGTERM and SIGINT into a clean stop through the server's new --stop-file, since the engine can't catch signals. The server tells everyone it shut down, logs its statistics and exits 0; the wrapper kills it only after 15 s. - --config=PATH reads options from a file (packaging/server/server.cfg); options after it on the command line win. - --health-port answers GET /health with JSON (phase, players, tick, uptime) on loopback by default. HealthServer is non-blocking and bounded. - packaging/docker: a debian:13 image that checks the engine against tools/engine.sha512, imports the game at build time, runs as an unprivileged user, and works with a read-only root and a small /tmp. compose.yaml is project and container tidewall-server, read-only, no capabilities. - packaging/systemd: a hardened unit with a dynamic user. - make server-package-check: compose up, healthy, non-root, read-only, a real headless client joins over UDP into the drop, then SIGTERM stops it cleanly. This also covers H7 (graceful shutdown on SIGTERM). Closes #38 Closes #70 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,10 +21,13 @@ var loop: ServerLoop
|
||||
var game: NetGame = null
|
||||
## Makes the server findable on the LAN and in list servers, when listening.
|
||||
var beacon: ServerBeacon = null
|
||||
## Answers GET /health, when asked for with --health-port.
|
||||
var health: HealthServer = null
|
||||
|
||||
var _errors: ErrorCounter = ErrorCounter.new()
|
||||
var _accumulator: float = 0.0
|
||||
var _started_usec: int = 0
|
||||
var _stop_checked_msec: int = 0
|
||||
var _finished: bool = false
|
||||
var _matches_ended: int = 0
|
||||
## When listening: microseconds each tick took including the network's share (after_tick:
|
||||
@@ -44,6 +47,9 @@ func _ready() -> void:
|
||||
_log("config_error", {"message": problem})
|
||||
_finish(2)
|
||||
return
|
||||
if not config.stop_file.is_empty() and FileAccess.file_exists(config.stop_file):
|
||||
# Left over from an earlier run; it isn't a request to stop this one.
|
||||
DirAccess.remove_absolute(config.stop_file)
|
||||
if not config.has_seed:
|
||||
config.match_seed = _fresh_seed()
|
||||
Engine.max_fps = 0 if config.unthrottled else REALTIME_MAX_FPS
|
||||
@@ -61,6 +67,15 @@ func _ready() -> void:
|
||||
return
|
||||
game = NetGame.new(loop, net)
|
||||
_open_beacon()
|
||||
if config.health_port > 0:
|
||||
health = HealthServer.new()
|
||||
health.report = _health_report
|
||||
var health_error: Error = health.listen(config.health_port, config.health_bind)
|
||||
if health_error != OK:
|
||||
_log("health_failed", {"port": config.health_port, "message": "couldn't listen for health checks on TCP port %d: %s" % [config.health_port, error_string(health_error)]})
|
||||
health = null
|
||||
_finish(2)
|
||||
return
|
||||
var started: Dictionary = config.to_dictionary()
|
||||
started["engine"] = Engine.get_version_info()["string"]
|
||||
started["tick_rate_hz"] = Tuning.TICK_RATE_HZ
|
||||
@@ -73,6 +88,12 @@ func _ready() -> void:
|
||||
func _process(delta: float) -> void:
|
||||
if _finished or loop == null:
|
||||
return
|
||||
if health != null:
|
||||
health.poll()
|
||||
if _stop_requested():
|
||||
_log("stopping", {"reason": "stop file"})
|
||||
_finish(0)
|
||||
return
|
||||
if game != null:
|
||||
game.poll()
|
||||
for event: Dictionary in game.events:
|
||||
@@ -108,6 +129,28 @@ func _exit_tree() -> void:
|
||||
_finish(0)
|
||||
|
||||
|
||||
## True when the stop file has appeared. Looked for a few times a second.
|
||||
func _stop_requested() -> bool:
|
||||
if config.stop_file.is_empty():
|
||||
return false
|
||||
var now: int = Time.get_ticks_msec()
|
||||
if now - _stop_checked_msec < 250:
|
||||
return false
|
||||
_stop_checked_msec = now
|
||||
return FileAccess.file_exists(config.stop_file)
|
||||
|
||||
|
||||
func _health_report() -> Dictionary:
|
||||
return {
|
||||
"status": "ok",
|
||||
"phase": loop.match_flow.phase_name(),
|
||||
"players": game.net.player_count() if game != null else 0,
|
||||
"max_players": loop.match_flow.rules.max_players,
|
||||
"tick": loop.ticks,
|
||||
"uptime_s": roundi((Time.get_ticks_usec() - _started_usec) / 1000000.0),
|
||||
}
|
||||
|
||||
|
||||
## True once the run has done what it was asked: its ticks, or its matches.
|
||||
func _done() -> bool:
|
||||
return (config.ticks > 0 and loop.ticks >= config.ticks) or (config.matches > 0 and _matches_ended >= config.matches)
|
||||
@@ -169,6 +212,8 @@ func _finish(requested_code: int) -> void:
|
||||
game.net.close()
|
||||
if beacon != null:
|
||||
beacon.close()
|
||||
if health != null:
|
||||
health.close()
|
||||
if loop != null:
|
||||
var stats: Dictionary = loop.timing_stats()
|
||||
stats["state_hash"] = loop.sim.state_hash()
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
## A tiny HTTP endpoint that says whether a dedicated server is up, for container health
|
||||
## checks, systemd watchers and monitoring. `GET /health` answers 200 with a JSON body:
|
||||
##
|
||||
## {"status":"ok","phase":"lobby","players":3,"max_players":24,"tick":1200,"uptime_s":40}
|
||||
##
|
||||
## Anything else gets 404, and a request that is malformed, too long or too slow is
|
||||
## closed. It listens on loopback unless told otherwise, holds a handful of connections
|
||||
## at most, and never blocks: call poll() once a frame.
|
||||
class_name HealthServer
|
||||
extends RefCounted
|
||||
|
||||
const MAX_CONNECTIONS: int = 8
|
||||
const MAX_REQUEST_BYTES: int = 2048
|
||||
const TIMEOUT_MSEC: int = 2000
|
||||
|
||||
## Fills in the body: func() -> Dictionary. "status" is added if missing.
|
||||
var report: Callable = func() -> Dictionary: return {}
|
||||
## Requests answered, by status code, for tests and logs.
|
||||
var answered: Dictionary = {}
|
||||
|
||||
var _server: TCPServer = TCPServer.new()
|
||||
## {"peer": StreamPeerTCP, "data": PackedByteArray, "since": int}
|
||||
var _connections: Array[Dictionary] = []
|
||||
|
||||
|
||||
func listen(port: int, address: String = "127.0.0.1") -> Error:
|
||||
return _server.listen(port, address)
|
||||
|
||||
|
||||
func port() -> int:
|
||||
return _server.get_local_port()
|
||||
|
||||
|
||||
func close() -> void:
|
||||
for connection: Dictionary in _connections:
|
||||
(connection["peer"] as StreamPeerTCP).disconnect_from_host()
|
||||
_connections.clear()
|
||||
_server.stop()
|
||||
|
||||
|
||||
func poll(now_msec: int = -1) -> void:
|
||||
var now: int = now_msec if now_msec >= 0 else Time.get_ticks_msec()
|
||||
while _server.is_connection_available():
|
||||
var peer: StreamPeerTCP = _server.take_connection()
|
||||
if _connections.size() >= MAX_CONNECTIONS:
|
||||
peer.disconnect_from_host()
|
||||
continue
|
||||
_connections.append({"peer": peer, "data": PackedByteArray(), "since": now})
|
||||
var still: Array[Dictionary] = []
|
||||
for connection: Dictionary in _connections:
|
||||
if not _serve(connection, now):
|
||||
still.append(connection)
|
||||
_connections = still
|
||||
|
||||
|
||||
## Reads what has come and answers once the request line and headers are in. Returns
|
||||
## true when the connection is finished with.
|
||||
func _serve(connection: Dictionary, now: int) -> bool:
|
||||
var peer: StreamPeerTCP = connection["peer"]
|
||||
peer.poll()
|
||||
if peer.get_status() != StreamPeerTCP.STATUS_CONNECTED:
|
||||
return true
|
||||
# Packed arrays are values: take a copy, add to it and put it back.
|
||||
var data: PackedByteArray = connection["data"]
|
||||
var available: int = peer.get_available_bytes()
|
||||
if available > 0:
|
||||
var got: Array = peer.get_partial_data(mini(available, MAX_REQUEST_BYTES))
|
||||
if int(got[0]) == OK:
|
||||
data.append_array(got[1])
|
||||
connection["data"] = data
|
||||
var text: String = data.get_string_from_ascii()
|
||||
if text.contains("\r\n\r\n") or text.contains("\n\n"):
|
||||
_answer(peer, text.get_slice("\n", 0).strip_edges())
|
||||
return true
|
||||
if data.size() >= MAX_REQUEST_BYTES or now - int(connection["since"]) > TIMEOUT_MSEC:
|
||||
peer.disconnect_from_host()
|
||||
return true
|
||||
return false
|
||||
|
||||
|
||||
func _answer(peer: StreamPeerTCP, request_line: String) -> void:
|
||||
var parts: PackedStringArray = request_line.split(" ")
|
||||
var code: int = 404
|
||||
var body: String = "{\"status\":\"not found\"}"
|
||||
if parts.size() == 3 and parts[0] == "GET" and (parts[1] == "/health" or parts[1] == "/health/"):
|
||||
code = 200
|
||||
var fields: Dictionary = report.call()
|
||||
if not fields.has("status"):
|
||||
fields["status"] = "ok"
|
||||
body = JSON.stringify(fields)
|
||||
answered[code] = int(answered.get(code, 0)) + 1
|
||||
var reason: String = "OK" if code == 200 else "Not Found"
|
||||
var head: String = "HTTP/1.0 %d %s\r\nContent-Type: application/json\r\nContent-Length: %d\r\nConnection: close\r\nCache-Control: no-store\r\n\r\n" % [code, reason, body.to_utf8_buffer().size()]
|
||||
peer.put_data((head + body).to_utf8_buffer())
|
||||
peer.disconnect_from_host()
|
||||
@@ -0,0 +1 @@
|
||||
uid://mpeamjfwg8n3
|
||||
@@ -4,6 +4,18 @@
|
||||
## [--bot-difficulty=mixed|easy|normal|hard] [--ticks=N]
|
||||
## [--unthrottled] [--log-every=N] [--matches=N] [--world-hash-every=N]
|
||||
## [--name=NAME] [--list=HOST[:PORT]]... [--no-lan]
|
||||
## [--health-port=N] [--health-bind=ADDRESS] [--stop-file=PATH]
|
||||
## [--config=PATH]
|
||||
##
|
||||
## `--config` reads options from a file, one per line, written without the leading
|
||||
## dashes: `port = 24650`, or a bare `listen` for a switch. Blank lines and lines
|
||||
## starting with # are skipped. Options after `--config` on the command line win.
|
||||
## See packaging/server/server.cfg.
|
||||
##
|
||||
## `--health-port` answers `GET /health` over HTTP on that TCP port (HealthServer), on
|
||||
## `--health-bind`, loopback by default. `--stop-file` makes the server stop cleanly,
|
||||
## telling everyone connected, as soon as that file exists: the packaging's wrapper
|
||||
## creates it on SIGTERM, which the engine itself can't catch.
|
||||
##
|
||||
## With `--listen` the server takes network players on UDP `--port`, waits in the lobby
|
||||
## for the first of them, and with any `--bots` above 0 fills each match's empty places
|
||||
@@ -55,6 +67,11 @@ var server_name: String = NetDiscovery.DEFAULT_SERVER_NAME
|
||||
var lists: Array[Dictionary] = []
|
||||
## Answer LAN discovery. On by default for a listening server.
|
||||
var lan: bool = true
|
||||
## TCP port for the health endpoint; 0 for none.
|
||||
var health_port: int = 0
|
||||
var health_bind: String = "127.0.0.1"
|
||||
## A file whose appearance asks the server to stop; empty for none.
|
||||
var stop_file: String = ""
|
||||
## Human-readable problems found while parsing. Empty when the config is usable.
|
||||
var errors: Array[String] = []
|
||||
|
||||
@@ -68,7 +85,12 @@ static func parse(args: PackedStringArray) -> ServerConfig:
|
||||
return config
|
||||
config.server = true
|
||||
for index: int in range(start + 1, args.size()):
|
||||
config._parse_one(args[index])
|
||||
var arg: String = args[index]
|
||||
if arg.begins_with("--config="):
|
||||
for from_file: String in config._read_file(arg.substr("--config=".length())):
|
||||
config._parse_one(from_file)
|
||||
else:
|
||||
config._parse_one(arg)
|
||||
config._validate()
|
||||
return config
|
||||
|
||||
@@ -147,6 +169,16 @@ func _parse_one(arg: String) -> void:
|
||||
errors.append("--bot-difficulty must be mixed, easy, normal or hard, got '%s'" % value)
|
||||
"--fault":
|
||||
fault = value
|
||||
"--health-port":
|
||||
if _check_int(key, value):
|
||||
health_port = value.to_int()
|
||||
"--health-bind":
|
||||
if not value.is_valid_ip_address():
|
||||
errors.append("--health-bind needs an IP address, got '%s'" % value)
|
||||
else:
|
||||
health_bind = value
|
||||
"--stop-file":
|
||||
stop_file = value
|
||||
"--name":
|
||||
server_name = NetDiscovery.clean_server_name(value)
|
||||
"--list":
|
||||
@@ -185,6 +217,30 @@ static func parse_host_port(text: String, default_port: int) -> Dictionary:
|
||||
return {"host": host, "port": port}
|
||||
|
||||
|
||||
## The options in a config file, as command-line arguments.
|
||||
func _read_file(path: String) -> Array[String]:
|
||||
var args: Array[String] = []
|
||||
var file: FileAccess = FileAccess.open(path, FileAccess.READ)
|
||||
if file == null:
|
||||
errors.append("couldn't read the config file '%s': %s" % [path, error_string(FileAccess.get_open_error())])
|
||||
return args
|
||||
var number: int = 0
|
||||
while not file.eof_reached():
|
||||
number += 1
|
||||
var line: String = file.get_line().strip_edges()
|
||||
if line.is_empty() or line.begins_with("#"):
|
||||
continue
|
||||
if line.begins_with("config"):
|
||||
errors.append("%s:%d: a config file can't include another" % [path, number])
|
||||
continue
|
||||
var at: int = line.find("=")
|
||||
if at < 0:
|
||||
args.append("--" + line)
|
||||
else:
|
||||
args.append("--%s=%s" % [line.substr(0, at).strip_edges(), line.substr(at + 1).strip_edges()])
|
||||
return args
|
||||
|
||||
|
||||
func _check_int(key: String, value: String) -> bool:
|
||||
if value.is_valid_int():
|
||||
return true
|
||||
@@ -214,5 +270,7 @@ func _validate() -> void:
|
||||
errors.append("--world-hash-every is for a server started with --listen")
|
||||
if not listen and (not lists.is_empty() or not lan):
|
||||
errors.append("--list and --no-lan are for a server started with --listen")
|
||||
if health_port < 0 or health_port > 65535:
|
||||
errors.append("--health-port must be between 1 and 65535, or 0 for none, got %d" % health_port)
|
||||
if fault not in FAULTS:
|
||||
errors.append("unknown --fault '%s'" % fault)
|
||||
|
||||
Reference in New Issue
Block a user