diff --git a/src/net/net_world.gd b/src/net/net_world.gd index 121df1d..2e03178 100644 --- a/src/net/net_world.gd +++ b/src/net/net_world.gd @@ -257,36 +257,74 @@ static func apply_own(player: SimEntity, own: Dictionary) -> void: ## up to some tick, and brought its pieces' building up to that tick, agrees with the ## server's simulation at that tick. static func world_hash(sim: Simulation) -> String: - var buffer: StreamPeerBuffer = StreamPeerBuffer.new() - buffer.big_endian = false - buffer.put_64(sim.tide.started_tick if sim.tide != null else -1) - if sim.tide != null: - buffer.put_data(sim.tide.levels.to_byte_array()) - buffer.put_64(sim.props.count()) - buffer.put_data(sim.props.health.to_byte_array()) - var piece_ids: Array = sim.structures.pieces.keys() - piece_ids.sort() - buffer.put_64(piece_ids.size()) - for piece_id: int in piece_ids: - var piece: StructureSet.Piece = sim.structures.pieces[piece_id] - buffer.put_64(piece.id) - buffer.put_u8(piece.kind) - buffer.put_u8(piece.material) - buffer.put_u8(piece.facing) - for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]: - buffer.put_32(value) - buffer.put_64(piece.owner) - buffer.put_32(piece.health) - buffer.put_32(piece.build_ticks_left) - buffer.put_u8(1 if piece.grounded else 0) - buffer.put_64(piece.placed_tick) - sim.loot.write_state(buffer) var hasher: HashingContext = HashingContext.new() hasher.start(HashingContext.HASH_SHA256) - hasher.update(buffer.data_array) + for part: PackedByteArray in _world_parts(sim): + hasher.update(part) return hasher.finish().hex_encode() +## The same state as world_hash(), hashed part by part ("tide", "props", "pieces", +## "loot"), with "piece_list": every piece as [id, health, build_ticks_left, grounded, +## placed_tick, kind, owner], in ID order. For telling where a client and the server +## disagree. +static func world_detail(sim: Simulation) -> Dictionary: + var parts: Array[PackedByteArray] = _world_parts(sim) + var detail: Dictionary = {} + var names: Array[String] = ["tide", "props", "pieces", "loot"] + for index: int in names.size(): + var hasher: HashingContext = HashingContext.new() + hasher.start(HashingContext.HASH_SHA256) + hasher.update(parts[index]) + detail[names[index]] = hasher.finish().hex_encode().substr(0, 12) + var piece_list: Array = [] + var piece_ids: Array = sim.structures.pieces.keys() + piece_ids.sort() + for piece_id: int in piece_ids: + var piece: StructureSet.Piece = sim.structures.pieces[piece_id] + piece_list.append([piece.id, piece.health, piece.build_ticks_left, 1 if piece.grounded else 0, piece.placed_tick, piece.kind, piece.owner]) + detail["piece_list"] = piece_list + return detail + + +## The bytes world_hash() hashes, in four parts: tide, props, pieces and loot. Their +## concatenation is the same buffer the hash has always covered. +static func _world_parts(sim: Simulation) -> Array[PackedByteArray]: + var tide: StreamPeerBuffer = _buffer() + tide.put_64(sim.tide.started_tick if sim.tide != null else -1) + if sim.tide != null: + tide.put_data(sim.tide.levels.to_byte_array()) + var props: StreamPeerBuffer = _buffer() + props.put_64(sim.props.count()) + props.put_data(sim.props.health.to_byte_array()) + var pieces: StreamPeerBuffer = _buffer() + var piece_ids: Array = sim.structures.pieces.keys() + piece_ids.sort() + pieces.put_64(piece_ids.size()) + for piece_id: int in piece_ids: + var piece: StructureSet.Piece = sim.structures.pieces[piece_id] + pieces.put_64(piece.id) + pieces.put_u8(piece.kind) + pieces.put_u8(piece.material) + pieces.put_u8(piece.facing) + for value: int in [piece.slot.x, piece.slot.y, piece.slot.z, piece.slot.w]: + pieces.put_32(value) + pieces.put_64(piece.owner) + pieces.put_32(piece.health) + pieces.put_32(piece.build_ticks_left) + pieces.put_u8(1 if piece.grounded else 0) + pieces.put_64(piece.placed_tick) + var loot: StreamPeerBuffer = _buffer() + sim.loot.write_state(loot) + return [tide.data_array, props.data_array, pieces.data_array, loot.data_array] + + +static func _buffer() -> StreamPeerBuffer: + var buffer: StreamPeerBuffer = StreamPeerBuffer.new() + buffer.big_endian = false + return buffer + + static func _put_varint(out: StreamPeerBuffer, value: int) -> void: value &= 0xFFFFFFFF while value >= 0x80: diff --git a/src/server/headless_server.gd b/src/server/headless_server.gd index aa25d6d..353777b 100644 --- a/src/server/headless_server.gd +++ b/src/server/headless_server.gd @@ -191,7 +191,7 @@ func _tick() -> void: game.after_tick() _net_tick_usec.append(Time.get_ticks_usec() - began) if config.world_hash_every > 0 and loop.sim.tick > 0 and loop.sim.tick % config.world_hash_every < Tuning.NET_SNAPSHOT_INTERVAL_TICKS: - _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim)}) + _log("world_hash", {"match": NetSnapshot.decode_match(game.match_notice())["index"], "sim_tick": loop.sim.tick, "hash": NetWorld.world_hash(loop.sim), "detail": NetWorld.world_detail(loop.sim)}) for event: Dictionary in loop.match_flow.events: if event["type"] in ["phase", "match_started", "match_ended"]: var fields: Dictionary = event.duplicate() diff --git a/test/net/test_world_sync.gd b/test/net/test_world_sync.gd index 515e44b..a0bda59 100644 --- a/test/net/test_world_sync.gd +++ b/test/net/test_world_sync.gd @@ -294,3 +294,23 @@ func test_the_client_refuses_what_the_server_would() -> void: assert_false(replica.dirty, "a world nothing was done to can be used again") assert_eq(_predict(_command(W)), Building.Result.OK) assert_true(replica.dirty) + + +func test_world_detail_names_the_part_that_differs() -> void: + _tick(_command(W)) + _deliver() + _agrees("placed") + var ours: Dictionary = NetWorld.world_detail(server) + var theirs: Dictionary = NetWorld.world_detail(world) + for part: String in ["tide", "props", "pieces", "loot"]: + assert_eq(ours[part], theirs[part], part) + assert_eq(str(ours["piece_list"]), str(theirs["piece_list"])) + assert_eq((ours["piece_list"] as Array).size(), 1) + # A building piece a tick ahead on the client: only the pieces differ. + replica.sync(server.tick + 1) + theirs = NetWorld.world_detail(world) + assert_eq(ours["props"], theirs["props"]) + assert_eq(ours["loot"], theirs["loot"]) + assert_ne(ours["pieces"], theirs["pieces"]) + assert_ne(str(ours["piece_list"]), str(theirs["piece_list"])) + assert_ne(NetWorld.world_hash(world), NetWorld.world_hash(server)) diff --git a/tools/soak_clients.gd b/tools/soak_clients.gd index ae471c5..1cdb612 100644 --- a/tools/soak_clients.gd +++ b/tools/soak_clients.gd @@ -11,7 +11,8 @@ ## client whose world has drifted is caught. ## ## Prints JSON lines: -## {"event":"hash", "client", "match", "sim_tick", "hash"}: the client's world hash on +## {"event":"hash", "client", "match", "sim_tick", "hash", "detail", "replica_tick"}: the +## client's world hash (with NetWorld.world_detail() and its ClientWorld's tick) on ## the snapshots where the server logs its own (--world-hash-every on the server, the ## same N as --hash-every here), when the client's world has caught up exactly. ## {"event":"client_match", "client", "match", ...}: a client's prediction statistics @@ -206,7 +207,7 @@ func _sample(client: Dictionary) -> void: if int(client["number"]) == _tamper and game.playing() and game.world.props.count() > 0: _tamper = -1 game.world.props.health[0] -= 1 - print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world)})) + print(JSON.stringify({"event": "hash", "client": client["number"], "match": game.match_index, "sim_tick": tick, "hash": NetWorld.world_hash(game.world), "detail": NetWorld.world_detail(game.world), "replica_tick": game.replica.tick})) func _finish() -> void: diff --git a/tools/soak_report.gd b/tools/soak_report.gd index 302c982..d3089e6 100644 --- a/tools/soak_report.gd +++ b/tools/soak_report.gd @@ -118,8 +118,10 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: result["client_mem_growth_bytes"] = _client_growth(clients) # The world, compared hash by hash. var server_hashes: Dictionary = {} + var server_detail: Dictionary = {} for record: Dictionary in _all(server, "world_hash"): server_hashes[Vector2i(int(record["match"]), int(record["sim_tick"]))] = String(record["hash"]) + server_detail[Vector2i(int(record["match"]), int(record["sim_tick"]))] = record.get("detail", {}) var compared: int = 0 var differ: int = 0 var per_client: Dictionary = {} @@ -132,7 +134,7 @@ func check(server: Array[Dictionary], clients: Array[Dictionary], client_count: if server_hashes[key] != String(record["hash"]): differ += 1 if differ <= 5: - _fail("client %d's world differs from the server's in match %d at tick %d" % [record["client"], key.x, key.y]) + _fail("client %d's world differs from the server's in match %d at tick %d (replica tick %s): %s" % [record["client"], key.x, key.y, record.get("replica_tick", "?"), describe_difference(server_detail.get(key, {}), record.get("detail", {}))]) result["hashes_compared"] = compared result["hashes_differing"] = differ if differ > 5: @@ -233,3 +235,40 @@ func _all(lines: Array[Dictionary], event: String) -> Array[Dictionary]: func _count(lines: Array[Dictionary], event: String) -> int: return _all(lines, event).size() + + +## Which parts of two NetWorld.world_detail() records differ, and for the pieces, each +## piece whose entry differs, as "id: server [...] client [...]" (at most eight). +static func describe_difference(server: Dictionary, client: Dictionary) -> String: + if server.is_empty() or client.is_empty(): + return "no detail logged" + var parts: Array[String] = [] + for name: String in ["tide", "props", "pieces", "loot"]: + if String(server.get(name, "")) != String(client.get(name, "")): + parts.append(name) + var notes: Array[String] = ["parts differing: %s" % (", ".join(parts) if not parts.is_empty() else "none")] + var theirs: Dictionary = {} + for entry: Array in client.get("piece_list", []): + theirs[int(entry[0])] = entry + var ours: Dictionary = {} + for entry: Array in server.get("piece_list", []): + ours[int(entry[0])] = entry + var ids: Dictionary = {} + for piece_id: int in ours: + ids[piece_id] = true + for piece_id: int in theirs: + ids[piece_id] = true + var sorted: Array = ids.keys() + sorted.sort() + var shown: int = 0 + for piece_id: int in sorted: + var server_entry: Variant = ours.get(piece_id) + var client_entry: Variant = theirs.get(piece_id) + if server_entry != null and client_entry != null and str(server_entry) == str(client_entry): + continue + shown += 1 + if shown <= 8: + notes.append("piece %d: server %s client %s" % [piece_id, str(server_entry), str(client_entry)]) + if shown > 8: + notes.append("and %d more pieces" % (shown - 8)) + return "; ".join(notes)