346 lines
18 KiB
TypeScript
346 lines
18 KiB
TypeScript
import { chmod, mkdir, readFile, stat } from 'node:fs/promises';
|
|
import { join } from 'node:path';
|
|
import { DatabaseSync } from 'node:sqlite';
|
|
import { z } from 'zod';
|
|
import { NETWORK, CONSENSUS, type TestnetBlock,
|
|
type NftState, type NftStateView, type TestnetTransaction, type MiningTemplate,
|
|
type TestnetDeck, type TestnetSnapshot, type TestnetSummary, type CanonicalPage } from '../../shared/testnet.ts';
|
|
import { CHAIN_ID, GENESIS_HASH, applyTransactionOverlay, blockSchema, makeTemplate,
|
|
applyTransferOverlay, materializeState, transactionId, transactionSchema, transferSchema,
|
|
validateBlock, workFor } from './consensus.ts';
|
|
import type { PowEngine } from './randomx.ts';
|
|
import type { Article } from '../../shared/protocol.ts';
|
|
import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
|
|
import { PROFILE } from '../../shared/network.ts';
|
|
import { RpcError } from '../resources.ts';
|
|
import { CanonicalIndex } from './canonical-index.ts';
|
|
|
|
const MAX_PENDING = 256;
|
|
const MAX_DECK_PAGE = 24;
|
|
const MAX_LEGACY_FILE = 128 * 1024 * 1024;
|
|
const diskSchema = z.object({ version: z.literal(1), network: z.literal(NETWORK),
|
|
chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(10_000),
|
|
pendingTransfers: z.array(transactionSchema).max(MAX_PENDING) }).strict();
|
|
type Node = { block: TestnetBlock; state: NftStateView; work: bigint };
|
|
const DATABASE_SCHEMA_VERSION = '1';
|
|
|
|
/** SQLite-backed DAG. Blocks append transactionally; pending rows are replaced transactionally.
|
|
* Each block state retains only its NFT delta, sharing the parent's state overlay.
|
|
*/
|
|
export class Ledger {
|
|
private readonly nodes = new Map<string, Node>();
|
|
private tip = GENESIS_HASH;
|
|
private pending: TestnetTransaction[] = [];
|
|
private pendingState: NftStateView = {};
|
|
private queue: Promise<unknown> = Promise.resolve();
|
|
private queued = 0;
|
|
/** Application-owned connectivity metadata; never part of consensus or persistence. */
|
|
peers = 0;
|
|
|
|
private readonly file: string;
|
|
private readonly db: DatabaseSync;
|
|
private readonly pow: PowEngine;
|
|
private readonly index: CanonicalIndex;
|
|
private closed = false;
|
|
private revision = 0;
|
|
private reorgs = 0;
|
|
private canonicalCache: { tip: string; blocks: TestnetBlock[] } | undefined;
|
|
private constructor(file: string, db: DatabaseSync, pow: PowEngine) {
|
|
this.file = file; this.db = db; this.pow = pow; this.index = new CanonicalIndex(db);
|
|
}
|
|
|
|
static async open(dir: string, pow: PowEngine): Promise<Ledger> {
|
|
const directory = join(dir, PROFILE);
|
|
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
const file = join(directory, 'ledger.sqlite');
|
|
let databaseExists = false;
|
|
try { databaseExists = (await stat(file)).size > 0; }
|
|
catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
|
|
const db = new DatabaseSync(file);
|
|
try {
|
|
const tableExists = (name: string) => Boolean(db.prepare(
|
|
"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
|
|
if (databaseExists && !tableExists('metadata')) {
|
|
throw new Error('SQLite ledger metadata missing; refusing to initialize over unknown data');
|
|
}
|
|
const readMeta = databaseExists ? db.prepare('SELECT value FROM metadata WHERE key = ?') : undefined;
|
|
if (!databaseExists) {
|
|
db.exec('BEGIN IMMEDIATE');
|
|
try {
|
|
db.exec(`CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
|
|
CREATE TABLE blocks (
|
|
sequence INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
hash TEXT NOT NULL UNIQUE,
|
|
previous_hash TEXT NOT NULL,
|
|
block_json TEXT NOT NULL
|
|
);
|
|
CREATE INDEX blocks_by_parent ON blocks(previous_hash);
|
|
CREATE TABLE pending_transfers (
|
|
position INTEGER PRIMARY KEY,
|
|
id TEXT NOT NULL UNIQUE,
|
|
transaction_json TEXT NOT NULL
|
|
);`);
|
|
const insertMeta = db.prepare('INSERT INTO metadata(key, value) VALUES(?, ?)');
|
|
insertMeta.run('schema_version', DATABASE_SCHEMA_VERSION);
|
|
insertMeta.run('network', NETWORK);
|
|
insertMeta.run('chain_id', CHAIN_ID);
|
|
db.exec('COMMIT');
|
|
} catch (error) { db.exec('ROLLBACK'); throw error; }
|
|
} else {
|
|
if (!tableExists('blocks') || !tableExists('pending_transfers')) {
|
|
throw new Error('SQLite ledger schema is incomplete; refusing to repair stored data automatically');
|
|
}
|
|
const schemaVersion = readMeta!.get('schema_version') as { value: string } | undefined;
|
|
const storedNetwork = readMeta!.get('network') as { value: string } | undefined;
|
|
const storedChain = readMeta!.get('chain_id') as { value: string } | undefined;
|
|
if (schemaVersion?.value !== DATABASE_SCHEMA_VERSION || storedNetwork?.value !== NETWORK || storedChain?.value !== CHAIN_ID) {
|
|
throw new Error('SQLite ledger schema or chain mismatch; refusing to reset stored data');
|
|
}
|
|
}
|
|
if (process.platform !== 'win32') await chmod(file, 0o600);
|
|
db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;');
|
|
|
|
const migrated = db.prepare('SELECT value FROM metadata WHERE key = ?').get('legacy_json_migrated') as { value: string } | undefined;
|
|
if (!migrated) {
|
|
const legacyFile = join(directory, 'ledger.json');
|
|
let legacy: unknown;
|
|
try {
|
|
const info = await stat(legacyFile);
|
|
if (info.size > MAX_LEGACY_FILE) throw new Error('Legacy testnet ledger exceeds 128 MiB; preserve it and migrate offline');
|
|
legacy = JSON.parse(await readFile(legacyFile, 'utf8'));
|
|
} catch (error) {
|
|
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
|
|
}
|
|
const insertMeta = db.prepare('INSERT OR REPLACE INTO metadata(key, value) VALUES(?, ?)');
|
|
db.exec('BEGIN IMMEDIATE');
|
|
try {
|
|
if (legacy !== undefined) {
|
|
const oldDisk = diskSchema.parse(legacy);
|
|
const insertBlock = db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)');
|
|
for (const block of oldDisk.blocks) insertBlock.run(block.hash, block.previousHash, JSON.stringify(block));
|
|
const insertPending = db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
|
|
oldDisk.pendingTransfers.forEach((tx, index) => insertPending.run(index, transactionId(tx), JSON.stringify(tx)));
|
|
}
|
|
insertMeta.run('legacy_json_migrated', '1');
|
|
db.exec('COMMIT');
|
|
} catch (error) { db.exec('ROLLBACK'); throw error; }
|
|
}
|
|
|
|
const ledger = new Ledger(file, db, pow);
|
|
for (const row of db.prepare('SELECT block_json FROM blocks ORDER BY sequence').iterate() as Iterable<{ block_json: string }>) {
|
|
const raw = blockSchema.parse(JSON.parse(row.block_json));
|
|
if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
|
|
const node = await ledger.validate(raw);
|
|
ledger.nodes.set(node.block.hash, node);
|
|
if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
|
|
}
|
|
let state = ledger.tipState();
|
|
const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transactionId)));
|
|
for (const row of db.prepare('SELECT id, transaction_json FROM pending_transfers ORDER BY position').iterate() as Iterable<{ id: string; transaction_json: string }>) {
|
|
const tx = transactionSchema.parse(JSON.parse(row.transaction_json));
|
|
const id = transactionId(tx);
|
|
if (row.id !== id || seen.has(id)) throw new Error('Duplicate or mismatched persisted transfer');
|
|
state = applyTransactionOverlay(state, tx, ledger.height() + 1);
|
|
seen.add(id);
|
|
ledger.pending.push(tx);
|
|
}
|
|
ledger.pendingState = state;
|
|
ledger.index.initialize(ledger.canonical(), materializeState(ledger.tipState()));
|
|
return ledger;
|
|
} catch (error) {
|
|
db.close();
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
get size(): number { return this.nodes.size; }
|
|
diagnostics() { const createdAt=this.nodes.get(this.tip)?.block.createdAt??null;
|
|
return { height:this.height(), dagBlocks:this.size, pending:this.pending.length, mutationQueue:this.queued,
|
|
reorgsSinceStartup:this.reorgs,tipCreatedAt:createdAt,tipAgeSeconds:createdAt===null?null:Math.max(0,Date.now()/1000-createdAt) }; }
|
|
get allBlocks(): TestnetBlock[] {
|
|
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
|
|
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
|
|
}
|
|
close(): void {
|
|
if (this.closed) return;
|
|
this.closed = true;
|
|
this.db.close();
|
|
}
|
|
block(hash: string): TestnetBlock | undefined {
|
|
const block = this.nodes.get(hash)?.block;
|
|
return block ? structuredClone(block) : undefined;
|
|
}
|
|
/** Append-only insertion order is topological, including all valid side branches.
|
|
* HTTP callers should request limit=32 and return next=offset+length or null.
|
|
*/
|
|
inventory(offset: number, limit: number): TestnetBlock[] {
|
|
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 1000) {
|
|
throw new Error('Invalid inventory range');
|
|
}
|
|
const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence LIMIT ? OFFSET ?')
|
|
.all(limit, offset) as { block_json: string }[];
|
|
return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
|
|
}
|
|
|
|
async submit(raw: unknown): Promise<{ accepted: boolean; reorg: boolean }> {
|
|
const input = structuredClone(raw);
|
|
return this.serialize(async () => {
|
|
const parsed = blockSchema.parse(input);
|
|
if (this.nodes.has(parsed.hash)) return { accepted: false, reorg: false };
|
|
const node = await this.validate(parsed);
|
|
const wins = node.work > this.tipWork();
|
|
const oldChain = this.canonical();
|
|
const chain = wins ? [...this.branch(node.block.previousHash), node.block] : oldChain;
|
|
const hashes = new Set(chain.map(b => b.hash));
|
|
const disconnected = wins ? oldChain.filter(b => !hashes.has(b.hash)) : [];
|
|
const filtered = wins ? this.filterPending(node.state, chain,
|
|
[...disconnected.flatMap(b => b.transactions), ...this.pending]) : undefined;
|
|
const pending = filtered?.pending ?? this.pending;
|
|
// Nothing visible changes until the atomic replacement succeeds.
|
|
const oldHashes = new Set(oldChain.map(b => b.hash));
|
|
const connected = wins ? chain.filter(b => !oldHashes.has(b.hash)) : [];
|
|
this.persist(node.block, pending, disconnected, connected);
|
|
this.nodes.set(node.block.hash, node);
|
|
if (wins) this.tip = node.block.hash;
|
|
if (disconnected.length) this.reorgs++;
|
|
this.pending = pending;
|
|
if (filtered) this.pendingState = filtered.state;
|
|
this.revision++;
|
|
return { accepted: true, reorg: disconnected.length > 0 };
|
|
});
|
|
}
|
|
|
|
async submitTransfer(raw: unknown): Promise<string> {
|
|
const tx = transferSchema.parse(structuredClone(raw));
|
|
return this.submitTransaction(tx);
|
|
}
|
|
|
|
async submitTransaction(raw: unknown): Promise<string> {
|
|
const input = structuredClone(raw);
|
|
return this.serialize(async () => {
|
|
const tx = transactionSchema.parse(input);
|
|
const id = transactionId(tx);
|
|
if (this.pending.some(t => transactionId(t) === id)) return id;
|
|
if (this.pending.length >= MAX_PENDING) throw new Error('Testnet mempool limit: 256 transfers');
|
|
const nextPendingState = applyTransactionOverlay(this.pendingState, tx, this.height() + 1);
|
|
const pending = [...this.pending, tx];
|
|
this.persist(undefined, pending);
|
|
this.pending = pending;
|
|
this.pendingState = nextPendingState;
|
|
this.revision++;
|
|
return id;
|
|
});
|
|
}
|
|
|
|
template(owner: string, publicKey: string, article: Article, statsSnapshot: MintStatsSnapshot): MiningTemplate {
|
|
return makeTemplate(this.canonical(), this.tipState(), owner, publicKey,
|
|
structuredClone(this.pending.slice(0, CONSENSUS.maxTransactions)), structuredClone(article),
|
|
structuredClone(statsSnapshot));
|
|
}
|
|
snapshot(): TestnetSnapshot {
|
|
const blocks = this.canonical();
|
|
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH,
|
|
tip: this.tip, cumulativeWork: this.tipWork().toString(), blocks,
|
|
transfers: blocks.flatMap(b => b.transactions), pendingTransfers: this.pending,
|
|
peers: this.peers, algorithm: 'RandomX',
|
|
maturity: CONSENSUS.coinbaseMaturity });
|
|
}
|
|
/** Small metadata/mempool response. No full-chain traversal or clone. */
|
|
summary(): TestnetSummary {
|
|
return { network: NETWORK, chainId: CHAIN_ID, genesis: GENESIS_HASH, tip: this.tip,
|
|
height: this.height(), revision: this.revision, cumulativeWork: this.tipWork().toString(),
|
|
pendingTransfers: structuredClone(this.pending), peers: this.peers, algorithm: 'RandomX', maturity: CONSENSUS.coinbaseMaturity };
|
|
}
|
|
peerStatus() { return { network: NETWORK, chainId: CHAIN_ID, size: this.size, tip: this.tip }; }
|
|
pendingTransactions(): TestnetTransaction[] { return structuredClone(this.pending); }
|
|
canonicalEvents(after: number, limit = 64) { return this.index.events(after, limit); }
|
|
canonicalHead(): number { return this.index.head(); }
|
|
nft(token: string) { return this.index.nft(token, this.height()); }
|
|
canonicalTransaction(id: string) { return this.index.transaction(id); }
|
|
legacySnapshot(maxBlocks: number): TestnetSnapshot {
|
|
// Reject before any full-chain copy/serialization. Never truncate ownership history silently.
|
|
if (this.height() > maxBlocks) throw new RpcError(413, 'Full snapshot disabled at this height; use /api/summary and /api/canonical pages.');
|
|
return this.snapshot();
|
|
}
|
|
canonicalPage(tip: string, offset: number, limit = 16): CanonicalPage {
|
|
if (tip !== this.tip) throw new RpcError(409, 'Canonical tip changed; restart paged loading.');
|
|
if (!Number.isSafeInteger(offset) || offset < 0 || offset > this.height() ||
|
|
!Number.isSafeInteger(limit) || limit < 1 || limit > 16) throw new RpcError(400, 'Invalid canonical page');
|
|
const blocks = structuredClone(this.canonical().slice(offset, offset + limit));
|
|
return { network: NETWORK, chainId: CHAIN_ID, tip: this.tip, height: this.height(), offset, blocks,
|
|
next: offset + blocks.length < this.height() ? offset + blocks.length : null };
|
|
}
|
|
deck(address: string, offset = 0, limit = MAX_DECK_PAGE): TestnetDeck {
|
|
if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_DECK_PAGE) {
|
|
throw new Error('Invalid deck page');
|
|
}
|
|
const { total, nfts } = this.index.deck(address, offset, limit, this.height());
|
|
const next = offset + nfts.length < total ? offset + nfts.length : null;
|
|
return structuredClone({ network: NETWORK, chainId: CHAIN_ID, address, tip: this.tip,
|
|
offset, limit, total, next, nfts });
|
|
}
|
|
|
|
private height(): number { return this.nodes.get(this.tip)?.block.height ?? 0; }
|
|
private tipState(): NftStateView { return this.nodes.get(this.tip)?.state ?? {}; }
|
|
private tipWork(): bigint { return this.nodes.get(this.tip)?.work ?? 0n; }
|
|
private branch(hash: string): TestnetBlock[] {
|
|
const result: TestnetBlock[] = [];
|
|
while (hash !== GENESIS_HASH) {
|
|
const node = this.nodes.get(hash);
|
|
if (!node) throw new Error('Unknown parent');
|
|
result.push(node.block);
|
|
hash = node.block.previousHash;
|
|
}
|
|
return result.reverse();
|
|
}
|
|
private canonical(): TestnetBlock[] {
|
|
if (this.canonicalCache?.tip !== this.tip) this.canonicalCache = { tip: this.tip, blocks: this.branch(this.tip) };
|
|
return this.canonicalCache.blocks; // Internal immutable block references only; public APIs clone their slice.
|
|
}
|
|
private async validate(raw: TestnetBlock): Promise<Node> {
|
|
const ancestors = this.branch(raw.previousHash);
|
|
const parent = this.nodes.get(raw.previousHash);
|
|
const { block, state } = await validateBlock(raw, ancestors, parent?.state ?? {}, this.pow);
|
|
return { block, state, work: (parent?.work ?? 0n) + workFor(block.target) };
|
|
}
|
|
private filterPending(parent: NftStateView, chain: TestnetBlock[], candidates: TestnetTransaction[]):
|
|
{ pending: TestnetTransaction[]; state: NftStateView } {
|
|
let state = parent;
|
|
const seen = new Set(chain.flatMap(b => b.transactions.map(transactionId)));
|
|
const pending: TestnetTransaction[] = [];
|
|
for (const tx of candidates) {
|
|
if (pending.length === MAX_PENDING) break;
|
|
const id = transactionId(tx);
|
|
if (seen.has(id)) continue;
|
|
try { state = applyTransactionOverlay(state, tx, chain.length + 1); }
|
|
catch { continue; } // Disconnected issuance, immature coins and conflicts are not replayable.
|
|
seen.add(id); pending.push(tx);
|
|
}
|
|
return { pending, state };
|
|
}
|
|
private serialize<T>(operation: () => Promise<T>): Promise<T> {
|
|
if (this.queued >= 256) return Promise.reject(new Error('Testnet mutation queue full'));
|
|
this.queued++;
|
|
const result = this.queue.then(operation);
|
|
this.queue = result.catch(() => undefined).finally(() => { this.queued--; });
|
|
return result;
|
|
}
|
|
private persist(block: TestnetBlock | undefined, pendingTransfers: TestnetTransaction[],
|
|
disconnected: TestnetBlock[] = [], connected: TestnetBlock[] = []): void {
|
|
this.db.exec('BEGIN IMMEDIATE');
|
|
try {
|
|
if (block) this.db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)')
|
|
.run(block.hash, block.previousHash, JSON.stringify(block));
|
|
this.db.exec('DELETE FROM pending_transfers');
|
|
const insertPending = this.db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
|
|
pendingTransfers.forEach((tx, index) => insertPending.run(index, transactionId(tx), JSON.stringify(tx)));
|
|
for (const old of [...disconnected].reverse()) this.index.disconnect(old);
|
|
for (const next of connected) this.index.connect(next);
|
|
this.db.exec('COMMIT');
|
|
} catch (error) {
|
|
this.db.exec('ROLLBACK');
|
|
throw error;
|
|
}
|
|
}
|
|
}
|