From 0ddcb48ea0214e7f5136eda87bd25135afd6b489 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 13 Nov 2026 11:46:35 -0500 Subject: [PATCH] H13: Say which part of the world differs when a soak hash does When a client's world hash differs from the server's, the soak report now names the part that differs (tide, props, pieces or loot), the client's ClientWorld tick, and each piece whose entry differs on the two sides, so the next transient mismatch can be traced. - NetWorld.world_detail(): part hashes and a piece list. world_hash() is unchanged: the same bytes, hashed in four parts. - The server's world_hash log lines and the soak clients' hash lines carry the detail. A full soak with this (24 clients, 2 matches) compared 3,743 hashes with 0 differing, so the mismatch didn't come back to trace yet. Refs #80 Co-Authored-By: Claude Opus 5.5 --- src/net/net_world.gd | 88 +++++++++++++++++++++++++---------- src/server/headless_server.gd | 2 +- test/net/test_world_sync.gd | 20 ++++++++ tools/soak_clients.gd | 5 +- tools/soak_report.gd | 41 +++++++++++++++- 5 files changed, 127 insertions(+), 29 deletions(-) diff --git a/src/net/net_world.gd b/src/net/net_world.gd index 121df1d..2e03178 100644 --- a/src/net/net_world.gd +++ b/src/net/net_world.gd @@ -257,36 +257,74 @@ static func apply_own(player: SimEntity, own: Dictionary) -> void: ## up to some tick, and brought its pieces' building up to that tick, agrees with the ## server's simulation at that tick. static func world_hash(sim: Simulation) -> String: - var buffer: StreamPeerBuffer = StreamPeerBuffer.new() - buffer.big_endian = false - buffer.put_64(sim.tide.started_tick if sim.tide != null else -1) - if sim.tide != null: - buffer.put_data(sim.tide.levels.to_byte_array()) - buffer.put_64(sim.props.count()) - buffer.put_data(sim.props.health.to_byte_array()) - var piece_ids: Array = sim.structures.pieces.keys() - piece_ids.sort() - buffer.put_64(piece_ids.size()) - for piece_id: int in piece_ids: - var piece: StructureSet.Piece = sim.structures.pieces[piece_id] - buffer.put_64(piece.id) - buffer.put_u8(piece.kind) - buffer.put_u8(piece.material) - buffer.put_u8(piece.facing) - for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]: - buffer.put_32(value) - buffer.put_64(piece.owner) - buffer.put_32(piece.health) - buffer.put_32(piece.build_ticks_left) - buffer.put_u8(1 if piece.grounded else 0) - buffer.put_64(piece.placed_tick) - sim.loot.write_state(buffer) var hasher: HashingContext = HashingContext.new() hasher.start(HashingContext.HASH_SHA256) - hasher.update(buffer.data_array) + for part: PackedByteArray in _world_parts(sim): + hasher.update(part) return hasher.finish().hex_encode() +## The same state as world_hash(), hashed part by part ("tide", "props", "pieces", +## "loot"), with "piece_list": every piece as [id, health, build_ticks_left, grounded, +## placed_tick, kind, owner], in ID order. For telling where a client and the server +## disagree. +static func world_detail(sim: Simulation) -> Dictionary: + var parts: Array[PackedByteArray] = _world_parts(sim) + var detail: Dictionary = {} + var names: Array[String] = ["tide", "props", "pieces", "loot"] + for index: int in names.size(): + var hasher: HashingContext = HashingContext.new() + hasher.start(HashingContext.HASH_SHA256) + hasher.update(parts[index]) + detail[names[index]] = hasher.finish().hex_encode().substr(0, 12) + var piece_list: Array = [] + var piece_ids: Array = sim.structures.pieces.keys() + piece_ids.sort() + for piece_id: int in piece_ids: + var piece: StructureSet.Piece = sim.structures.pieces[piece_id] + piece_list.append([piece.id, piece.health, piece.build_ticks_left, 1 if piece.grounded else 0, piece.placed_tick, piece.kind, piece.owner]) + detail["piece_list"] = piece_list + return detail + + +## The bytes world_hash() hashes, in four parts: tide, props, pieces and loot. Their +## concatenation is the same buffer the hash has always covered. +static func _world_parts(sim: Simulation) -> Array[PackedByteArray]: + var tide: StreamPeerBuffer = _buffer() + tide.put_64(sim.tide.started_tick if sim.tide != null else -1) + if sim.tide != null: + tide.put_data(sim.tide.levels.to_byte_array()) + var props: StreamPeerBuffer = _buffer() + props.put_64(sim.props.count()) + props.put_data(sim.props.health.to_byte_array()) + var pieces: StreamPeerBuffer = _buffer() + var piece_ids: Array = sim.structures.pieces.keys() + piece_ids.sort() + pieces.put_64(piece_ids.size()) + for piece_id: int in piece_ids: + var piece: StructureSet.Piece = sim.structures.pieces[piece_id] + pieces.put_64(piece.id) + pieces.put_u8(piece.kind) + pieces.put_u8(piece.material) + pieces.put_u8(piece.facing) + for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]: + pieces.put_32(value) + pieces.put_64(piece.owner) + pieces.put_32(piece.health) + pieces.put_32(piece.build_ticks_left) + pieces.put_u8(1 if piece.grounded else 0) + pieces.put_64(piece.placed_tick) + var loot: StreamPeerBuffer = _buffer() + sim.loot.write_state(loot) + return [tide.data_array, props.data_array, pieces.data_array, loot.data_array] + + +static func _buffer() -> StreamPeerBuffer: + var buffer: StreamPeerBuffer = StreamPeerBuffer.new() + buffer.big_endian = false + return buffer + + static func _put_varint(out: StreamPeerBuffer, value: int) -> void: value &= 0xFFFFFFFF while value >= 0x80: diff --git a/src/server/headless_server.gd b/src/server/headless_server.gd index aa25d6d..353777b 100644 --- a/src/server/headless_server.gd +++ b/src/server/headless_server.gd @@ -191,7 +191,7 @@ func _tick() -> void: game.after_tick() _net_tick_usec.append(Time.get_ticks_usec() - began) if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS: - _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim)}) + _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)}) for event: Dictionary in loop.match_flow.events: if event["type"] in ["phase", "match_started", "match_ended"]: var fields: Dictionary = event.duplicate() diff --git a/test/net/test_world_sync.gd b/test/net/test_world_sync.gd index 515e44b..a0bda59 100644 --- a/test/net/test_world_sync.gd +++ b/test/net/test_world_sync.gd @@ -294,3 +294,23 @@ func test_the_client_refuses_what_the_server_would() -> void: assert_false(replica.dirty, "a world nothing was done to can be used again") assert_eq(_predict(_command(W)), Building.Result.OK) assert_true(replica.dirty) + + +func test_world_detail_names_the_part_that_differs() -> void: + _tick(_command(W)) + _deliver() + _agrees("placed") + var ours: Dictionary = NetWorld.world_detail(server) + var theirs: Dictionary = NetWorld.world_detail(world) + for part: String in ["tide", "props", "pieces", "loot"]: + assert_eq(ours[part], theirs[part], part) + assert_eq(str(ours["piece_list"]), str(theirs["piece_list"])) + assert_eq((ours["piece_list"] as Array).size(), 1) + # A building piece a tick ahead on the client: only the pieces differ. + replica.sync(server.tick + 1) + theirs = NetWorld.world_detail(world) + assert_eq(ours["props"], theirs["props"]) + assert_eq(ours["loot"], theirs["loot"]) + assert_ne(ours["pieces"], theirs["pieces"]) + assert_ne(str(ours["piece_list"]), str(theirs["piece_list"])) + assert_ne(NetWorld.world_hash(world), NetWorld.world_hash(server)) diff --git a/tools/soak_clients.gd b/tools/soak_clients.gd index ae471c5..1cdb612 100644 --- a/tools/soak_clients.gd +++ b/tools/soak_clients.gd @@ -11,7 +11,8 @@ ## client whose world has drifted is caught. ## ## Prints JSON lines: -## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on +## {"event":"hash", "client", "match", "sim_tick", "hash", "detail", "replica_tick"}: the +## client's world hash (with NetWorld.world_detail() and its ClientWorld's tick) on ## the snapshots where the server logs its own (--world-hash-every on the server, the ## same N as --hash-every here), when the client's world has caught up exactly. ## {"event":"client_match", "client", "match", ...}: a client's prediction statistics @@ -206,7 +207,7 @@ func _sample(client: Dictionary) -> void: if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0: _tamper = -1 game.world.props.health[0] -= 1 - print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)})) + print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world), "detail": NetWorld.world_detail(game.world), "replica_tick": game.replica.tick})) func _finish() -> void: diff --git a/tools/soak_report.gd b/tools/soak_report.gd index 302c982..d3089e6 100644 --- a/tools/soak_report.gd +++ b/tools/soak_report.gd @@ -118,8 +118,10 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: result["client_mem_growth_bytes"] = _client_growth(clients) # The world, compared hash by hash. var server_hashes: Dictionary = {} + var server_detail: Dictionary = {} for record: Dictionary in _all(server, "world_hash"): server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"]) + server_detail[Vector2i(int(record["match"]), int(record["sim_tick"]))] = record.get("detail", {}) var compared: int = 0 var differ: int = 0 var per_client: Dictionary = {} @@ -132,7 +134,7 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: if server_hashes[key] != String(record["hash"]): differ += 1 if differ <= 5: - _fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y]) + _fail("client %d's world differs from the server's in match %d at tick %d (replica tick %s): %s" % [record["client"], key.x, key.y, record.get("replica_tick", "?"), describe_difference(server_detail.get(key, {}), record.get("detail", {}))]) result["hashes_compared"] = compared result["hashes_differing"] = differ if differ > 5: @@ -233,3 +235,40 @@ func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]: func _count(lines: Array[Dictionary], event: String) -> int: return _all(lines, event).size() + + +## Which parts of two NetWorld.world_detail() records differ, and for the pieces, each +## piece whose entry differs, as "id: server [...] client [...]" (at most eight). +static func describe_difference(server: Dictionary, client: Dictionary) -> String: + if server.is_empty() or client.is_empty(): + return "no detail logged" + var parts: Array[String] = [] + for name: String in ["tide", "props", "pieces", "loot"]: + if String(server.get(name, "")) != String(client.get(name, "")): + parts.append(name) + var notes: Array[String] = ["parts differing: %s" % (", ".join(parts) if not parts.is_empty() else "none")] + var theirs: Dictionary = {} + for entry: Array in client.get("piece_list", []): + theirs[int(entry[0])] = entry + var ours: Dictionary = {} + for entry: Array in server.get("piece_list", []): + ours[int(entry[0])] = entry + var ids: Dictionary = {} + for piece_id: int in ours: + ids[piece_id] = true + for piece_id: int in theirs: + ids[piece_id] = true + var sorted: Array = ids.keys() + sorted.sort() + var shown: int = 0 + for piece_id: int in sorted: + var server_entry: Variant = ours.get(piece_id) + var client_entry: Variant = theirs.get(piece_id) + if server_entry != null and client_entry != null and str(server_entry) == str(client_entry): + continue + shown += 1 + if shown <= 8: + notes.append("piece %d: server %s client %s" % [piece_id, str(server_entry), str(client_entry)]) + if shown > 8: + notes.append("and %d more pieces" % (shown - 8)) + return "; ".join(notes)