WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

More scalable data storage

Browse Source
This commit is contained in:
eric
2026-10-19 20:17:10 -04:00
parent 0b9061372b
commit baad477ca5
16 changed files with 346 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/testnet/ledger.ts
+149 -86
View File
@@ -1,88 +1,155 @@
import { mkdir, open, rename, unlink } from 'node:fs/promises';
import { chmod, mkdir, readFile, stat } from 'node:fs/promises';
import { join } from 'node:path';
import { randomUUID } from 'node:crypto';
import { setTimeout as delay } from 'node:timers/promises';
import { DatabaseSync } from 'node:sqlite';
import { z } from 'zod';
import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer, type NftState,
type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate,
import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer,
type NftStateView, type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
import { CHAIN_ID, GENESIS_HASH, applyTransferOverlay, blockSchema, makeTemplate,
transferId, transferSchema, validateBlock, workFor } from './consensus.ts';
import type { PowEngine } from './randomx.ts';
import type { Article } from '../../shared/protocol.ts';
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
const MAX_BLOCKS = 10_000;
const MAX_PENDING = 256;
const MAX_FILE = 128 * 1024 * 1024;
const MAX_LEGACY_FILE = 128 * 1024 * 1024;
const diskSchema = z.object({ version: z.literal(1), network: z.literal(NETWORK),
chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(MAX_BLOCKS),
chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(10_000),
pendingTransfers: z.array(transferSchema).max(MAX_PENDING) }).strict();
type Node = { block: TestnetBlock; state: NftState; work: bigint };
type Node = { block: TestnetBlock; state: NftStateView; work: bigint };
const DATABASE_SCHEMA_VERSION = '1';
/** Experimental, bounded DAG. The supplied engine is always used, including on restart.
* One process must own a data directory; there is no cross-process writer locking.
/** SQLite-backed DAG. Blocks append transactionally; pending rows are replaced transactionally.
* Each block state retains only its NFT delta, sharing the parent's state overlay.
*/
export class Ledger {
private readonly nodes = new Map<string, Node>();
private tip = GENESIS_HASH;
private pending: NftTransfer[] = [];
private pendingState: NftStateView = {};
private queue: Promise<unknown> = Promise.resolve();
private queued = 0;
/** Application-owned connectivity metadata; never part of consensus or persistence. */
peers = 0;
private readonly file: string;
private readonly db: DatabaseSync;
private readonly pow: PowEngine;
private constructor(file: string, pow: PowEngine) { this.file = file; this.pow = pow; }
private closed = false;
private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }
static async open(dir: string, pow: PowEngine): Promise<Ledger> {
const directory = join(dir, 'testnet');
await mkdir(directory, { recursive: true, mode: 0o700 });
const ledger = new Ledger(join(directory, 'ledger.json'), pow);
let handle;
try { handle = await open(ledger.file, 'r'); }
catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
await ledger.persist([], []);
return ledger;
}
let text: string;
const file = join(directory, 'ledger.sqlite');
let databaseExists = false;
try { databaseExists = (await stat(file)).size > 0; }
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
const db = new DatabaseSync(file);
try {
if ((await handle.stat()).size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
// Bounded reads also protect against a file growing after stat().
const chunks: Buffer[] = [];
let size = 0;
for (;;) {
const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_FILE + 1 - size));
const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
if (!bytesRead) break;
size += bytesRead;
if (size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
chunks.push(buffer.subarray(0, bytesRead));
const tableExists = (name: string) => Boolean(db.prepare(
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
if (databaseExists && !tableExists('metadata')) {
throw new Error('SQLite ledger metadata missing; refusing to initialize over unknown data');
}
text = Buffer.concat(chunks).toString('utf8');
} finally { await handle.close(); }
const disk = diskSchema.parse(JSON.parse(text)); // Mismatches/corruption fail closed, never reset.
for (const raw of disk.blocks) {
if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
const node = await ledger.validate(raw);
ledger.nodes.set(node.block.hash, node);
if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
const readMeta = databaseExists ? db.prepare('SELECT value FROM metadata WHERE key = ?') : undefined;
if (!databaseExists) {
db.exec('BEGIN IMMEDIATE');
try {
db.exec(`CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
CREATE TABLE blocks (
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
hash TEXT NOT NULL UNIQUE,
previous_hash TEXT NOT NULL,
block_json TEXT NOT NULL
);
CREATE INDEX blocks_by_parent ON blocks(previous_hash);
CREATE TABLE pending_transfers (
position INTEGER PRIMARY KEY,
id TEXT NOT NULL UNIQUE,
transaction_json TEXT NOT NULL
);`);
const insertMeta = db.prepare('INSERT INTO metadata(key, value) VALUES(?, ?)');
insertMeta.run('schema_version', DATABASE_SCHEMA_VERSION);
insertMeta.run('network', NETWORK);
insertMeta.run('chain_id', CHAIN_ID);
db.exec('COMMIT');
} catch (error) { db.exec('ROLLBACK'); throw error; }
} else {
if (!tableExists('blocks') || !tableExists('pending_transfers')) {
throw new Error('SQLite ledger schema is incomplete; refusing to repair stored data automatically');
}
const schemaVersion = readMeta!.get('schema_version') as { value: string } | undefined;
const storedNetwork = readMeta!.get('network') as { value: string } | undefined;
const storedChain = readMeta!.get('chain_id') as { value: string } | undefined;
if (schemaVersion?.value !== DATABASE_SCHEMA_VERSION || storedNetwork?.value !== NETWORK || storedChain?.value !== CHAIN_ID) {
throw new Error('SQLite ledger schema or chain mismatch; refusing to reset stored data');
}
}
if (process.platform !== 'win32') await chmod(file, 0o600);
db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;');
const migrated = db.prepare('SELECT value FROM metadata WHERE key = ?').get('legacy_json_migrated') as { value: string } | undefined;
if (!migrated) {
const legacyFile = join(directory, 'ledger.json');
let legacy: unknown;
try {
const info = await stat(legacyFile);
if (info.size > MAX_LEGACY_FILE) throw new Error('Legacy testnet ledger exceeds 128 MiB; preserve it and migrate offline');
legacy = JSON.parse(await readFile(legacyFile, 'utf8'));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
}
const insertMeta = db.prepare('INSERT OR REPLACE INTO metadata(key, value) VALUES(?, ?)');
db.exec('BEGIN IMMEDIATE');
try {
if (legacy !== undefined) {
const oldDisk = diskSchema.parse(legacy);
const insertBlock = db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)');
for (const block of oldDisk.blocks) insertBlock.run(block.hash, block.previousHash, JSON.stringify(block));
const insertPending = db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
oldDisk.pendingTransfers.forEach((tx, index) => insertPending.run(index, transferId(tx), JSON.stringify(tx)));
}
insertMeta.run('legacy_json_migrated', '1');
db.exec('COMMIT');
} catch (error) { db.exec('ROLLBACK'); throw error; }
}
const ledger = new Ledger(file, db, pow);
for (const row of db.prepare('SELECT block_json FROM blocks ORDER BY sequence').iterate() as Iterable<{ block_json: string }>) {
const raw = blockSchema.parse(JSON.parse(row.block_json));
if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
const node = await ledger.validate(raw);
ledger.nodes.set(node.block.hash, node);
if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
}
let state = ledger.tipState();
const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transferId)));
for (const row of db.prepare('SELECT id, transaction_json FROM pending_transfers ORDER BY position').iterate() as Iterable<{ id: string; transaction_json: string }>) {
const tx = transferSchema.parse(JSON.parse(row.transaction_json));
const id = transferId(tx);
if (row.id !== id || seen.has(id)) throw new Error('Duplicate or mismatched persisted transfer');
state = applyTransferOverlay(state, tx, ledger.height() + 1);
seen.add(id);
ledger.pending.push(tx);
}
ledger.pendingState = state;
return ledger;
} catch (error) {
db.close();
throw error;
}
const state = structuredClone(ledger.tipState());
const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transferId)));
for (const tx of disk.pendingTransfers) {
const id = transferId(tx);
if (seen.has(id)) throw new Error('Duplicate persisted transfer');
applyTransfer(state, tx, ledger.height() + 1);
seen.add(id);
}
ledger.pending = disk.pendingTransfers;
return ledger;
}
get size(): number { return this.nodes.size; }
get allBlocks(): TestnetBlock[] { return this.inventory(0, MAX_BLOCKS); }
get allBlocks(): TestnetBlock[] {
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
}
close(): void {
if (this.closed) return;
this.closed = true;
this.db.close();
}
block(hash: string): TestnetBlock | undefined {
const block = this.nodes.get(hash)?.block;
return block ? structuredClone(block) : undefined;
@@ -91,10 +158,12 @@ export class Ledger {
* HTTP callers should request limit=32 and return next=offset+length or null.
*/
inventory(offset: number, limit: number): TestnetBlock[] {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_BLOCKS) {
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 1000) {
throw new Error('Invalid inventory range');
}
return structuredClone(Array.from(this.nodes.values()).slice(offset, offset + limit).map(n => n.block));
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence LIMIT ? OFFSET ?')
.all(limit, offset) as { block_json: string }[];
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
}
async submit(raw: unknown): Promise<{ accepted: boolean; reorg: boolean }> {
@@ -102,20 +171,21 @@ export class Ledger {
return this.serialize(async () => {
const parsed = blockSchema.parse(input);
if (this.nodes.has(parsed.hash)) return { accepted: false, reorg: false };
if (this.size >= MAX_BLOCKS) throw new Error('Experimental testnet limit: 10000 stored blocks');
const node = await this.validate(parsed);
const wins = node.work > this.tipWork();
const oldChain = this.canonical();
const chain = wins ? [...this.branch(node.block.previousHash), node.block] : oldChain;
const hashes = new Set(chain.map(b => b.hash));
const disconnected = wins ? oldChain.filter(b => !hashes.has(b.hash)) : [];
const pending = wins ? this.filterPending(node.state, chain,
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : this.pending;
const filtered = wins ? this.filterPending(node.state, chain,
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : undefined;
const pending = filtered?.pending ?? this.pending;
// Nothing visible changes until the atomic replacement succeeds.
await this.persist([...Array.from(this.nodes.values(), n => n.block), node.block], pending);
await this.persist(node.block, pending);
this.nodes.set(node.block.hash, node);
if (wins) this.tip = node.block.hash;
this.pending = pending;
if (filtered) this.pendingState = filtered.state;
return { accepted: true, reorg: disconnected.length > 0 };
});
}
@@ -127,12 +197,11 @@ export class Ledger {
const id = transferId(tx);
if (this.pending.some(t => transferId(t) === id)) return id;
if (this.pending.length >= MAX_PENDING) throw new Error('Testnet mempool limit: 256 transfers');
const state = structuredClone(this.tipState());
for (const pending of this.pending) applyTransfer(state, pending, this.height() + 1);
applyTransfer(state, tx, this.height() + 1); // No speculative issuance.
const nextPendingState = applyTransferOverlay(this.pendingState, tx, this.height() + 1);
const pending = [...this.pending, tx];
await this.persist(Array.from(this.nodes.values(), n => n.block), pending);
await this.persist(undefined, pending);
this.pending = pending;
this.pendingState = nextPendingState;
return id;
});
}
@@ -152,7 +221,7 @@ export class Ledger {
}
private height(): number { return this.nodes.get(this.tip)?.block.height ?? 0; }
private tipState(): NftState { return this.nodes.get(this.tip)?.state ?? {}; }
private tipState(): NftStateView { return this.nodes.get(this.tip)?.state ?? {}; }
private tipWork(): bigint { return this.nodes.get(this.tip)?.work ?? 0n; }
private branch(hash: string): TestnetBlock[] {
const result: TestnetBlock[] = [];
@@ -171,19 +240,20 @@ export class Ledger {
const { block, state } = await validateBlock(raw, ancestors, parent?.state ?? {}, this.pow);
return { block, state, work: (parent?.work ?? 0n) + workFor(block.target) };
}
private filterPending(parent: NftState, chain: TestnetBlock[], candidates: NftTransfer[]): NftTransfer[] {
const state = structuredClone(parent);
private filterPending(parent: NftStateView, chain: TestnetBlock[], candidates: NftTransfer[]):
{ pending: NftTransfer[]; state: NftStateView } {
let state = parent;
const seen = new Set(chain.flatMap(b => b.transactions.map(transferId)));
const pending: NftTransfer[] = [];
for (const tx of candidates) {
if (pending.length === MAX_PENDING) break;
const id = transferId(tx);
if (seen.has(id)) continue;
try { applyTransfer(state, tx, chain.length + 1); }
try { state = applyTransferOverlay(state, tx, chain.length + 1); }
catch { continue; } // Disconnected issuance, immature coins and conflicts are not replayable.
seen.add(id); pending.push(tx);
}
return pending;
return { pending, state };
}
private serialize<T>(operation: () => Promise<T>): Promise<T> {
if (this.queued >= 256) return Promise.reject(new Error('Testnet mutation queue full'));
@@ -192,25 +262,18 @@ export class Ledger {
this.queue = result.catch(() => undefined).finally(() => { this.queued--; });
return result;
}
private async persist(blocks: TestnetBlock[], pendingTransfers: NftTransfer[]): Promise<void> {
const data = JSON.stringify({ version: 1, network: NETWORK, chainId: CHAIN_ID, blocks, pendingTransfers });
if (Buffer.byteLength(data) > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
const temp = `${this.file}.${randomUUID()}.tmp`;
private async persist(block: TestnetBlock | undefined, pendingTransfers: NftTransfer[]): Promise<void> {
this.db.exec('BEGIN IMMEDIATE');
try {
const handle = await open(temp, 'wx', 0o600);
try { await handle.writeFile(data); await handle.sync(); }
finally { await handle.close(); }
// Windows scanners can briefly hold the destination open without delete sharing.
// Retry only transient sharing failures; never unlink the old durable ledger.
for (let attempt = 0; ; attempt++) {
try { await rename(temp, this.file); break; }
catch (error) {
const code = (error as NodeJS.ErrnoException).code;
if (process.platform !== 'win32' || attempt >= 5 ||
!['EPERM', 'EACCES', 'EBUSY'].includes(code ?? '')) throw error;
await delay(10 * 2 ** attempt);
}
}
} finally { await unlink(temp).catch(() => undefined); }
if (block) this.db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)')
.run(block.hash, block.previousHash, JSON.stringify(block));
this.db.exec('DELETE FROM pending_transfers');
const insertPending = this.db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
pendingTransfers.forEach((tx, index) => insertPending.run(index, transferId(tx), JSON.stringify(tx)));
this.db.exec('COMMIT');
} catch (error) {
this.db.exec('ROLLBACK');
throw error;
}
}
}