H13: Say which part of the world differs when a soak hash does
When a client's world hash differs from the server's, the soak report now names the part that differs (tide, props, pieces or loot), the client's ClientWorld tick, and each piece whose entry differs on the two sides, so the next transient mismatch can be traced. - NetWorld.world_detail(): part hashes and a piece list. world_hash() is unchanged: the same bytes, hashed in four parts. - The server's world_hash log lines and the soak clients' hash lines carry the detail. A full soak with this (24 clients, 2 matches) compared 3,743 hashes with 0 differing, so the mismatch didn't come back to trace yet. Refs #80 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+63
-25
@@ -257,36 +257,74 @@ static func apply_own(player: SimEntity, own: Dictionary) -> void:
|
|||||||
## up to some tick, and brought its pieces' building up to that tick, agrees with the
|
## up to some tick, and brought its pieces' building up to that tick, agrees with the
|
||||||
## server's simulation at that tick.
|
## server's simulation at that tick.
|
||||||
static func world_hash(sim: Simulation) -> String:
|
static func world_hash(sim: Simulation) -> String:
|
||||||
var buffer: StreamPeerBuffer = StreamPeerBuffer.new()
|
|
||||||
buffer.big_endian = false
|
|
||||||
buffer.put_64(sim.tide.started_tick if sim.tide != null else -1)
|
|
||||||
if sim.tide != null:
|
|
||||||
buffer.put_data(sim.tide.levels.to_byte_array())
|
|
||||||
buffer.put_64(sim.props.count())
|
|
||||||
buffer.put_data(sim.props.health.to_byte_array())
|
|
||||||
var piece_ids: Array = sim.structures.pieces.keys()
|
|
||||||
piece_ids.sort()
|
|
||||||
buffer.put_64(piece_ids.size())
|
|
||||||
for piece_id: int in piece_ids:
|
|
||||||
var piece: StructureSet.Piece = sim.structures.pieces[piece_id]
|
|
||||||
buffer.put_64(piece.id)
|
|
||||||
buffer.put_u8(piece.kind)
|
|
||||||
buffer.put_u8(piece.material)
|
|
||||||
buffer.put_u8(piece.facing)
|
|
||||||
for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]:
|
|
||||||
buffer.put_32(value)
|
|
||||||
buffer.put_64(piece.owner)
|
|
||||||
buffer.put_32(piece.health)
|
|
||||||
buffer.put_32(piece.build_ticks_left)
|
|
||||||
buffer.put_u8(1 if piece.grounded else 0)
|
|
||||||
buffer.put_64(piece.placed_tick)
|
|
||||||
sim.loot.write_state(buffer)
|
|
||||||
var hasher: HashingContext = HashingContext.new()
|
var hasher: HashingContext = HashingContext.new()
|
||||||
hasher.start(HashingContext.HASH_SHA256)
|
hasher.start(HashingContext.HASH_SHA256)
|
||||||
hasher.update(buffer.data_array)
|
for part: PackedByteArray in _world_parts(sim):
|
||||||
|
hasher.update(part)
|
||||||
return hasher.finish().hex_encode()
|
return hasher.finish().hex_encode()
|
||||||
|
|
||||||
|
|
||||||
|
## The same state as world_hash(), hashed part by part ("tide", "props", "pieces",
|
||||||
|
## "loot"), with "piece_list": every piece as [id, health, build_ticks_left, grounded,
|
||||||
|
## placed_tick, kind, owner], in ID order. For telling where a client and the server
|
||||||
|
## disagree.
|
||||||
|
static func world_detail(sim: Simulation) -> Dictionary:
|
||||||
|
var parts: Array[PackedByteArray] = _world_parts(sim)
|
||||||
|
var detail: Dictionary = {}
|
||||||
|
var names: Array[String] = ["tide", "props", "pieces", "loot"]
|
||||||
|
for index: int in names.size():
|
||||||
|
var hasher: HashingContext = HashingContext.new()
|
||||||
|
hasher.start(HashingContext.HASH_SHA256)
|
||||||
|
hasher.update(parts[index])
|
||||||
|
detail[names[index]] = hasher.finish().hex_encode().substr(0, 12)
|
||||||
|
var piece_list: Array = []
|
||||||
|
var piece_ids: Array = sim.structures.pieces.keys()
|
||||||
|
piece_ids.sort()
|
||||||
|
for piece_id: int in piece_ids:
|
||||||
|
var piece: StructureSet.Piece = sim.structures.pieces[piece_id]
|
||||||
|
piece_list.append([piece.id, piece.health, piece.build_ticks_left, 1 if piece.grounded else 0, piece.placed_tick, piece.kind, piece.owner])
|
||||||
|
detail["piece_list"] = piece_list
|
||||||
|
return detail
|
||||||
|
|
||||||
|
|
||||||
|
## The bytes world_hash() hashes, in four parts: tide, props, pieces and loot. Their
|
||||||
|
## concatenation is the same buffer the hash has always covered.
|
||||||
|
static func _world_parts(sim: Simulation) -> Array[PackedByteArray]:
|
||||||
|
var tide: StreamPeerBuffer = _buffer()
|
||||||
|
tide.put_64(sim.tide.started_tick if sim.tide != null else -1)
|
||||||
|
if sim.tide != null:
|
||||||
|
tide.put_data(sim.tide.levels.to_byte_array())
|
||||||
|
var props: StreamPeerBuffer = _buffer()
|
||||||
|
props.put_64(sim.props.count())
|
||||||
|
props.put_data(sim.props.health.to_byte_array())
|
||||||
|
var pieces: StreamPeerBuffer = _buffer()
|
||||||
|
var piece_ids: Array = sim.structures.pieces.keys()
|
||||||
|
piece_ids.sort()
|
||||||
|
pieces.put_64(piece_ids.size())
|
||||||
|
for piece_id: int in piece_ids:
|
||||||
|
var piece: StructureSet.Piece = sim.structures.pieces[piece_id]
|
||||||
|
pieces.put_64(piece.id)
|
||||||
|
pieces.put_u8(piece.kind)
|
||||||
|
pieces.put_u8(piece.material)
|
||||||
|
pieces.put_u8(piece.facing)
|
||||||
|
for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]:
|
||||||
|
pieces.put_32(value)
|
||||||
|
pieces.put_64(piece.owner)
|
||||||
|
pieces.put_32(piece.health)
|
||||||
|
pieces.put_32(piece.build_ticks_left)
|
||||||
|
pieces.put_u8(1 if piece.grounded else 0)
|
||||||
|
pieces.put_64(piece.placed_tick)
|
||||||
|
var loot: StreamPeerBuffer = _buffer()
|
||||||
|
sim.loot.write_state(loot)
|
||||||
|
return [tide.data_array, props.data_array, pieces.data_array, loot.data_array]
|
||||||
|
|
||||||
|
|
||||||
|
static func _buffer() -> StreamPeerBuffer:
|
||||||
|
var buffer: StreamPeerBuffer = StreamPeerBuffer.new()
|
||||||
|
buffer.big_endian = false
|
||||||
|
return buffer
|
||||||
|
|
||||||
|
|
||||||
static func _put_varint(out: StreamPeerBuffer, value: int) -> void:
|
static func _put_varint(out: StreamPeerBuffer, value: int) -> void:
|
||||||
value &= 0xFFFFFFFF
|
value &= 0xFFFFFFFF
|
||||||
while value >= 0x80:
|
while value >= 0x80:
|
||||||
|
|||||||
@@ -191,7 +191,7 @@ func _tick() -> void:
|
|||||||
game.after_tick()
|
game.after_tick()
|
||||||
_net_tick_usec.append(Time.get_ticks_usec() - began)
|
_net_tick_usec.append(Time.get_ticks_usec() - began)
|
||||||
if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS:
|
||||||
_log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim)})
|
_log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)})
|
||||||
for event: Dictionary in loop.match_flow.events:
|
for event: Dictionary in loop.match_flow.events:
|
||||||
if event["type"] in ["phase", "match_started", "match_ended"]:
|
if event["type"] in ["phase", "match_started", "match_ended"]:
|
||||||
var fields: Dictionary = event.duplicate()
|
var fields: Dictionary = event.duplicate()
|
||||||
|
|||||||
@@ -294,3 +294,23 @@ func test_the_client_refuses_what_the_server_would() -> void:
|
|||||||
assert_false(replica.dirty, "a world nothing was done to can be used again")
|
assert_false(replica.dirty, "a world nothing was done to can be used again")
|
||||||
assert_eq(_predict(_command(W)), Building.Result.OK)
|
assert_eq(_predict(_command(W)), Building.Result.OK)
|
||||||
assert_true(replica.dirty)
|
assert_true(replica.dirty)
|
||||||
|
|
||||||
|
|
||||||
|
func test_world_detail_names_the_part_that_differs() -> void:
|
||||||
|
_tick(_command(W))
|
||||||
|
_deliver()
|
||||||
|
_agrees("placed")
|
||||||
|
var ours: Dictionary = NetWorld.world_detail(server)
|
||||||
|
var theirs: Dictionary = NetWorld.world_detail(world)
|
||||||
|
for part: String in ["tide", "props", "pieces", "loot"]:
|
||||||
|
assert_eq(ours[part], theirs[part], part)
|
||||||
|
assert_eq(str(ours["piece_list"]), str(theirs["piece_list"]))
|
||||||
|
assert_eq((ours["piece_list"] as Array).size(), 1)
|
||||||
|
# A building piece a tick ahead on the client: only the pieces differ.
|
||||||
|
replica.sync(server.tick + 1)
|
||||||
|
theirs = NetWorld.world_detail(world)
|
||||||
|
assert_eq(ours["props"], theirs["props"])
|
||||||
|
assert_eq(ours["loot"], theirs["loot"])
|
||||||
|
assert_ne(ours["pieces"], theirs["pieces"])
|
||||||
|
assert_ne(str(ours["piece_list"]), str(theirs["piece_list"]))
|
||||||
|
assert_ne(NetWorld.world_hash(world), NetWorld.world_hash(server))
|
||||||
|
|||||||
@@ -11,7 +11,8 @@
|
|||||||
## client whose world has drifted is caught.
|
## client whose world has drifted is caught.
|
||||||
##
|
##
|
||||||
## Prints JSON lines:
|
## Prints JSON lines:
|
||||||
## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on
|
## {"event":"hash", "client", "match", "sim_tick", "hash", "detail", "replica_tick"}: the
|
||||||
|
## client's world hash (with NetWorld.world_detail() and its ClientWorld's tick) on
|
||||||
## the snapshots where the server logs its own (--world-hash-every on the server, the
|
## the snapshots where the server logs its own (--world-hash-every on the server, the
|
||||||
## same N as --hash-every here), when the client's world has caught up exactly.
|
## same N as --hash-every here), when the client's world has caught up exactly.
|
||||||
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
|
## {"event":"client_match", "client", "match", ...}: a client's prediction statistics
|
||||||
@@ -206,7 +207,7 @@ func _sample(client: Dictionary) -> void:
|
|||||||
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
|
if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0:
|
||||||
_tamper = -1
|
_tamper = -1
|
||||||
game.world.props.health[0] -= 1
|
game.world.props.health[0] -= 1
|
||||||
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)}))
|
print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world), "detail": NetWorld.world_detail(game.world), "replica_tick": game.replica.tick}))
|
||||||
|
|
||||||
|
|
||||||
func _finish() -> void:
|
func _finish() -> void:
|
||||||
|
|||||||
+40
-1
@@ -118,8 +118,10 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count:
|
|||||||
result["client_mem_growth_bytes"] = _client_growth(clients)
|
result["client_mem_growth_bytes"] = _client_growth(clients)
|
||||||
# The world, compared hash by hash.
|
# The world, compared hash by hash.
|
||||||
var server_hashes: Dictionary = {}
|
var server_hashes: Dictionary = {}
|
||||||
|
var server_detail: Dictionary = {}
|
||||||
for record: Dictionary in _all(server, "world_hash"):
|
for record: Dictionary in _all(server, "world_hash"):
|
||||||
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
|
server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"])
|
||||||
|
server_detail[Vector2i(int(record["match"]), int(record["sim_tick"]))] = record.get("detail", {})
|
||||||
var compared: int = 0
|
var compared: int = 0
|
||||||
var differ: int = 0
|
var differ: int = 0
|
||||||
var per_client: Dictionary = {}
|
var per_client: Dictionary = {}
|
||||||
@@ -132,7 +134,7 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count:
|
|||||||
if server_hashes[key] != String(record["hash"]):
|
if server_hashes[key] != String(record["hash"]):
|
||||||
differ += 1
|
differ += 1
|
||||||
if differ <= 5:
|
if differ <= 5:
|
||||||
_fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y])
|
_fail("client %d's world differs from the server's in match %d at tick %d (replica tick %s): %s" % [record["client"], key.x, key.y, record.get("replica_tick", "?"), describe_difference(server_detail.get(key, {}), record.get("detail", {}))])
|
||||||
result["hashes_compared"] = compared
|
result["hashes_compared"] = compared
|
||||||
result["hashes_differing"] = differ
|
result["hashes_differing"] = differ
|
||||||
if differ > 5:
|
if differ > 5:
|
||||||
@@ -233,3 +235,40 @@ func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]:
|
|||||||
|
|
||||||
func _count(lines: Array[Dictionary], event: String) -> int:
|
func _count(lines: Array[Dictionary], event: String) -> int:
|
||||||
return _all(lines, event).size()
|
return _all(lines, event).size()
|
||||||
|
|
||||||
|
|
||||||
|
## Which parts of two NetWorld.world_detail() records differ, and for the pieces, each
|
||||||
|
## piece whose entry differs, as "id: server [...] client [...]" (at most eight).
|
||||||
|
static func describe_difference(server: Dictionary, client: Dictionary) -> String:
|
||||||
|
if server.is_empty() or client.is_empty():
|
||||||
|
return "no detail logged"
|
||||||
|
var parts: Array[String] = []
|
||||||
|
for name: String in ["tide", "props", "pieces", "loot"]:
|
||||||
|
if String(server.get(name, "")) != String(client.get(name, "")):
|
||||||
|
parts.append(name)
|
||||||
|
var notes: Array[String] = ["parts differing: %s" % (", ".join(parts) if not parts.is_empty() else "none")]
|
||||||
|
var theirs: Dictionary = {}
|
||||||
|
for entry: Array in client.get("piece_list", []):
|
||||||
|
theirs[int(entry[0])] = entry
|
||||||
|
var ours: Dictionary = {}
|
||||||
|
for entry: Array in server.get("piece_list", []):
|
||||||
|
ours[int(entry[0])] = entry
|
||||||
|
var ids: Dictionary = {}
|
||||||
|
for piece_id: int in ours:
|
||||||
|
ids[piece_id] = true
|
||||||
|
for piece_id: int in theirs:
|
||||||
|
ids[piece_id] = true
|
||||||
|
var sorted: Array = ids.keys()
|
||||||
|
sorted.sort()
|
||||||
|
var shown: int = 0
|
||||||
|
for piece_id: int in sorted:
|
||||||
|
var server_entry: Variant = ours.get(piece_id)
|
||||||
|
var client_entry: Variant = theirs.get(piece_id)
|
||||||
|
if server_entry != null and client_entry != null and str(server_entry) == str(client_entry):
|
||||||
|
continue
|
||||||
|
shown += 1
|
||||||
|
if shown <= 8:
|
||||||
|
notes.append("piece %d: server %s client %s" % [piece_id, str(server_entry), str(client_entry)])
|
||||||
|
if shown > 8:
|
||||||
|
notes.append("and %d more pieces" % (shown - 8))
|
||||||
|
return "; ".join(notes)
|
||||||
|
|||||||
Reference in New Issue
Block a user