More scalable data storage

server/testnet/consensus.ts

@@ -1,7 +1,8 @@
 import { createHash, createPublicKey, verify } from 'node:crypto';
 import { z } from 'zod';
 import { NETWORK, CONSENSUS, headerMessage, blockClaimMessage, nftTransferMessage, statsSnapshotTuple,
-  type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type MiningTemplate } from '../../shared/testnet.ts';
+  type BlockHeader, type TestnetBlock, type NftTransfer, type NftState, type NftStateLayer,
+  type NftStateView, type MiningTemplate } from '../../shared/testnet.ts';
 import type { Article } from '../../shared/protocol.ts';
 import { deriveStats, type MintStatsSnapshot } from '../../shared/enrichment.ts';
 import type { PowEngine } from './randomx.ts';
@@ -70,7 +71,7 @@ const statsSnapshotSchema = z.object({ capturedAt: z.string().datetime(), revisi
       ctx.addIssue({ code: 'custom', message: 'Derived statistics do not match the page metrics', path: ['stats'] });
     }
   });
-export const blockSchema = z.object({ version: z.literal(4), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
+export const blockSchema = z.object({ version: z.literal(5), network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
   height: integer, previousHash: hex, createdAt: integer, article: articleSchema, statsSnapshot: statsSnapshotSchema, owner: hex,
   publicKey: key, nonce: integer, target: hex, key: hex, transactionsRoot: hex, stateRoot: hex,
   hash: hex, tokenId: hex, powHash: hex, signature, transactions: z.array(transferSchema).max(CONSENSUS.maxTransactions) }).strict();
@@ -102,9 +103,32 @@ export function merkleRoot(leaves: string[]): string {
   // Commit the count as well: odd-leaf padding must not make [a,b,c] equal [a,b,c,c].
   return sha256(JSON.stringify([NETWORK,'merkle',leaves.length,layer[0]]));
 }
-export function stateRoot(state: NftState): string {
-  return merkleRoot(Object.keys(state).sort().map(id => {
-    const n = state[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
+export function materializeState(state: NftStateView): NftState {
+  const layers: NftState[] = [];
+  let cursor: NftStateView = state;
+  while (isStateLayer(cursor)) {
+    layers.push(cursor.changes);
+    cursor = cursor.parent;
+  }
+  const result: NftState = Object.assign(Object.create(null) as NftState, cursor as NftState);
+  for (let i = layers.length - 1; i >= 0; i--) Object.assign(result, layers[i]);
+  return result;
+}
+function stateEntry(state: NftStateView, id: string): NftState[string] | undefined {
+  let cursor: NftStateView = state;
+  while (isStateLayer(cursor)) {
+    if (Object.hasOwn(cursor.changes, id)) return cursor.changes[id];
+    cursor = cursor.parent;
+  }
+  return (cursor as NftState)[id];
+}
+function isStateLayer(state: NftStateView): state is NftStateLayer {
+  return (state as NftStateLayer).kind === 'layer';
+}
+export function stateRoot(state: NftStateView): string {
+  const materialized = materializeState(state);
+  return merkleRoot(Object.keys(materialized).sort().map(id => {
+    const n = materialized[id]; return sha256(JSON.stringify([id, n.owner, n.sequence, n.mintedHeight, n.pageId]));
   }));
 }
 export function mintId(h: Pick<BlockHeader, 'previousHash' | 'height' | 'owner' | 'article' | 'statsSnapshot'>): string {
@@ -143,35 +167,49 @@ export function applyTransfer(state: NftState, tx: NftTransfer, height: number):
   if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
   state[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
 }
-export function transition(parent: NftState, header: BlockHeader, txs: NftTransfer[]): NftState {
-  const next = structuredClone(parent);
+export function applyTransferOverlay(state: NftStateView, tx: NftTransfer, height: number): NftStateLayer {
+  if (!transferSchema.safeParse(tx).success) throw new Error('Invalid transfer encoding/network');
+  const nft = stateEntry(state, tx.cardHash);
+  if (!nft || nft.owner !== tx.from || tx.from === tx.to || nft.sequence !== tx.sequence) {
+    throw new Error('Invalid NFT owner/sequence');
+  }
+  if (height - nft.mintedHeight < CONSENSUS.coinbaseMaturity) throw new Error('NFT coinbase is immature');
+  if (!verifySignature(tx.publicKey, tx.from, nftTransferMessage(tx), tx.signature)) throw new Error('Invalid transfer signature');
+  const changes: NftState = Object.create(null) as NftState;
+  changes[tx.cardHash] = { ...nft, owner: tx.to, sequence: tx.sequence + 1 };
+  return { kind: 'layer', parent: state, changes };
+}
+export function transition(parent: NftStateView, header: BlockHeader, txs: NftTransfer[]): NftStateLayer {
+  let next = parent;
   const ids = new Set<string>();
   for (const tx of txs) {
     const id = transferId(tx);
     if (ids.has(id)) throw new Error('Duplicate transfer');
-    ids.add(id); applyTransfer(next, tx, header.height);
+    ids.add(id);
+    next = applyTransferOverlay(next, tx, header.height);
   }
   const id = mintId(header);
-  if (next[id]) throw new Error('Duplicate NFT issuance');
-  next[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
-  return next;
+  if (stateEntry(next, id)) throw new Error('Duplicate NFT issuance');
+  const changes: NftState = Object.create(null) as NftState;
+  changes[id] = { owner: header.owner, sequence: 0, mintedHeight: header.height, pageId: header.article.pageId };
+  return { kind: 'layer', parent: next, changes };
 }
-export function makeTemplate(ancestors: TestnetBlock[], state: NftState, owner: string,
+export function makeTemplate(ancestors: TestnetBlock[], state: NftStateView, owner: string,
   publicKey: string, transactions: NftTransfer[], article: Article, statsSnapshot: MintStatsSnapshot,
   now = Math.floor(Date.now()/1000)): MiningTemplate {
   if (!hex.safeParse(owner).success || address(publicKey) !== owner) throw new Error('Invalid miner identity');
   const parsedArticle = articleSchema.parse(article);
   const parsedStats = statsSnapshotSchema.parse(statsSnapshot);
   const previousHash = ancestors.at(-1)?.hash ?? GENESIS_HASH;
-  const header: BlockHeader = { version: 4, network: NETWORK, chainId: CHAIN_ID,
+  const header: BlockHeader = { version: 5, network: NETWORK, chainId: CHAIN_ID,
     height: ancestors.length + 1, previousHash, createdAt: Math.max(now, medianTime(ancestors) + 1),
     article: parsedArticle, statsSnapshot: parsedStats, owner, publicKey, nonce: 0, target: expectedTarget(ancestors),
     key: expectedKey(ancestors, ancestors.length + 1), transactionsRoot: merkleRoot(transactions.map(transferId)), stateRoot: '' };
   header.stateRoot = stateRoot(transition(state, header, transactions));
   return { header, transactions, blob: miningBlob(header).toString('hex'), nonceOffset: 32 };
 }
-export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftState,
-  pow: PowEngine, now = Math.floor(Date.now()/1000)): Promise<{block:TestnetBlock;state:NftState}> {
+export async function validateBlock(raw: unknown, ancestors: TestnetBlock[], parentState: NftStateView,
+  pow: PowEngine, now = Math.floor(Date.now()/1000)): Promise<{block:TestnetBlock;state:NftStateView}> {
   if (Buffer.byteLength(JSON.stringify(raw)) > CONSENSUS.maxBlockBytes) throw new Error('Block size limit exceeded');
   const b = blockSchema.parse(raw);
   if (b.height !== ancestors.length + 1 || b.previousHash !== (ancestors.at(-1)?.hash ?? GENESIS_HASH)) throw new Error('Invalid parent/height');

server/testnet/http.ts

@@ -30,7 +30,7 @@ catch(e) {engine.close();await lock.close();await unlink(lockPath);throw e;}
 let peers: PeerNetwork;
 try { peers = new PeerNetwork({ ledger, urls:(process.env.WIKIDECK_PEERS ?? '').split(',').map(s=>s.trim()).filter(Boolean),
   allowPrivate:process.env.WIKIDECK_ALLOW_PRIVATE_PEERS === '1' }); }
-catch(e) {engine.close();await lock.close();await unlink(lockPath);throw e;}
+catch(e) {ledger.close();engine.close();await lock.close();await unlink(lockPath);throw e;}
 const app = express();
 app.disable('x-powered-by');
 app.use(helmet({contentSecurityPolicy:{directives:{defaultSrc:["'self'"],scriptSrc:["'self'"],
@@ -63,7 +63,7 @@ app.get('/api/wikipedia/:pageId', rateLimit({windowMs:60000,limit:60,standardHea
 app.get('/p2p/status',(_req,res)=>res.json({network:NETWORK,chainId:CHAIN_ID,size:ledger.size,tip:ledger.snapshot().tip}));
 app.get('/p2p/blocks',(req,res)=>{
   const offset=Number(req.query.offset??0);
-  if(!Number.isSafeInteger(offset)||offset<0||offset>10000){res.status(400).json({error:'Invalid offset'});return;}
+  if(!Number.isSafeInteger(offset)||offset<0){res.status(400).json({error:'Invalid offset'});return;}
   const blocks=ledger.inventory(offset,32);
   res.json({blocks,next:offset+blocks.length<ledger.size?offset+blocks.length:null});
 });
@@ -119,6 +119,6 @@ let stopping=false;
 for(const signal of ['SIGTERM','SIGINT']as const)process.once(signal,()=>{
   if(stopping)return;stopping=true;peers.close();
   const timeout=setTimeout(()=>process.exit(1),25000);timeout.unref();
-  server.close(async()=>{engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
+  server.close(async()=>{ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});clearTimeout(timeout);process.exit(0);});
 });
-server.on('error',async()=>{peers.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});
+server.on('error',async()=>{peers.close();ledger.close();engine.close();await lock.close();await unlink(lockPath).catch(()=>{});process.exit(1);});

server/testnet/ledger.ts

@@ -1,88 +1,155 @@
-import { mkdir, open, rename, unlink } from 'node:fs/promises';
+import { chmod, mkdir, readFile, stat } from 'node:fs/promises';
 import { join } from 'node:path';
-import { randomUUID } from 'node:crypto';
-import { setTimeout as delay } from 'node:timers/promises';
+import { DatabaseSync } from 'node:sqlite';
 import { z } from 'zod';
-import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer, type NftState,
-  type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
-import { CHAIN_ID, GENESIS_HASH, applyTransfer, blockSchema, makeTemplate,
+import { NETWORK, CONSENSUS, type TestnetBlock, type NftTransfer,
+  type NftStateView, type MiningTemplate, type TestnetSnapshot } from '../../shared/testnet.ts';
+import { CHAIN_ID, GENESIS_HASH, applyTransferOverlay, blockSchema, makeTemplate,
   transferId, transferSchema, validateBlock, workFor } from './consensus.ts';
 import type { PowEngine } from './randomx.ts';
 import type { Article } from '../../shared/protocol.ts';
 import type { MintStatsSnapshot } from '../../shared/enrichment.ts';
 
-const MAX_BLOCKS = 10_000;
 const MAX_PENDING = 256;
-const MAX_FILE = 128 * 1024 * 1024;
+const MAX_LEGACY_FILE = 128 * 1024 * 1024;
 const diskSchema = z.object({ version: z.literal(1), network: z.literal(NETWORK),
-  chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(MAX_BLOCKS),
+  chainId: z.literal(CHAIN_ID), blocks: z.array(blockSchema).max(10_000),
   pendingTransfers: z.array(transferSchema).max(MAX_PENDING) }).strict();
-type Node = { block: TestnetBlock; state: NftState; work: bigint };
+type Node = { block: TestnetBlock; state: NftStateView; work: bigint };
+const DATABASE_SCHEMA_VERSION = '1';
 
-/** Experimental, bounded DAG. The supplied engine is always used, including on restart.
- * One process must own a data directory; there is no cross-process writer locking.
+/** SQLite-backed DAG. Blocks append transactionally; pending rows are replaced transactionally.
+ * Each block state retains only its NFT delta, sharing the parent's state overlay.
  */
 export class Ledger {
   private readonly nodes = new Map<string, Node>();
   private tip = GENESIS_HASH;
   private pending: NftTransfer[] = [];
+  private pendingState: NftStateView = {};
   private queue: Promise<unknown> = Promise.resolve();
   private queued = 0;
   /** Application-owned connectivity metadata; never part of consensus or persistence. */
   peers = 0;
 
   private readonly file: string;
+  private readonly db: DatabaseSync;
   private readonly pow: PowEngine;
-  private constructor(file: string, pow: PowEngine) { this.file = file; this.pow = pow; }
+  private closed = false;
+  private constructor(file: string, db: DatabaseSync, pow: PowEngine) { this.file = file; this.db = db; this.pow = pow; }
 
   static async open(dir: string, pow: PowEngine): Promise<Ledger> {
     const directory = join(dir, 'testnet');
     await mkdir(directory, { recursive: true, mode: 0o700 });
-    const ledger = new Ledger(join(directory, 'ledger.json'), pow);
-    let handle;
-    try { handle = await open(ledger.file, 'r'); }
-    catch (error) {
-      if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
-      await ledger.persist([], []);
-      return ledger;
-    }
-    let text: string;
+    const file = join(directory, 'ledger.sqlite');
+    let databaseExists = false;
+    try { databaseExists = (await stat(file)).size > 0; }
+    catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; }
+    const db = new DatabaseSync(file);
     try {
-      if ((await handle.stat()).size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
-      // Bounded reads also protect against a file growing after stat().
-      const chunks: Buffer[] = [];
-      let size = 0;
-      for (;;) {
-        const buffer = Buffer.alloc(Math.min(64 * 1024, MAX_FILE + 1 - size));
-        const { bytesRead } = await handle.read(buffer, 0, buffer.length, null);
-        if (!bytesRead) break;
-        size += bytesRead;
-        if (size > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
-        chunks.push(buffer.subarray(0, bytesRead));
+      const tableExists = (name: string) => Boolean(db.prepare(
+        "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?").get(name));
+      if (databaseExists && !tableExists('metadata')) {
+        throw new Error('SQLite ledger metadata missing; refusing to initialize over unknown data');
       }
-      text = Buffer.concat(chunks).toString('utf8');
-    } finally { await handle.close(); }
-    const disk = diskSchema.parse(JSON.parse(text)); // Mismatches/corruption fail closed, never reset.
-    for (const raw of disk.blocks) {
-      if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
-      const node = await ledger.validate(raw);
-      ledger.nodes.set(node.block.hash, node);
-      if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
+      const readMeta = databaseExists ? db.prepare('SELECT value FROM metadata WHERE key = ?') : undefined;
+      if (!databaseExists) {
+        db.exec('BEGIN IMMEDIATE');
+        try {
+          db.exec(`CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL) WITHOUT ROWID;
+            CREATE TABLE blocks (
+              sequence INTEGER PRIMARY KEY AUTOINCREMENT,
+              hash TEXT NOT NULL UNIQUE,
+              previous_hash TEXT NOT NULL,
+              block_json TEXT NOT NULL
+            );
+            CREATE INDEX blocks_by_parent ON blocks(previous_hash);
+            CREATE TABLE pending_transfers (
+              position INTEGER PRIMARY KEY,
+              id TEXT NOT NULL UNIQUE,
+              transaction_json TEXT NOT NULL
+            );`);
+          const insertMeta = db.prepare('INSERT INTO metadata(key, value) VALUES(?, ?)');
+          insertMeta.run('schema_version', DATABASE_SCHEMA_VERSION);
+          insertMeta.run('network', NETWORK);
+          insertMeta.run('chain_id', CHAIN_ID);
+          db.exec('COMMIT');
+        } catch (error) { db.exec('ROLLBACK'); throw error; }
+      } else {
+        if (!tableExists('blocks') || !tableExists('pending_transfers')) {
+          throw new Error('SQLite ledger schema is incomplete; refusing to repair stored data automatically');
+        }
+        const schemaVersion = readMeta!.get('schema_version') as { value: string } | undefined;
+        const storedNetwork = readMeta!.get('network') as { value: string } | undefined;
+        const storedChain = readMeta!.get('chain_id') as { value: string } | undefined;
+        if (schemaVersion?.value !== DATABASE_SCHEMA_VERSION || storedNetwork?.value !== NETWORK || storedChain?.value !== CHAIN_ID) {
+          throw new Error('SQLite ledger schema or chain mismatch; refusing to reset stored data');
+        }
+      }
+      if (process.platform !== 'win32') await chmod(file, 0o600);
+      db.exec('PRAGMA journal_mode = WAL; PRAGMA synchronous = FULL; PRAGMA foreign_keys = ON; PRAGMA busy_timeout = 5000;');
+
+      const migrated = db.prepare('SELECT value FROM metadata WHERE key = ?').get('legacy_json_migrated') as { value: string } | undefined;
+      if (!migrated) {
+        const legacyFile = join(directory, 'ledger.json');
+        let legacy: unknown;
+        try {
+          const info = await stat(legacyFile);
+          if (info.size > MAX_LEGACY_FILE) throw new Error('Legacy testnet ledger exceeds 128 MiB; preserve it and migrate offline');
+          legacy = JSON.parse(await readFile(legacyFile, 'utf8'));
+        } catch (error) {
+          if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
+        }
+        const insertMeta = db.prepare('INSERT OR REPLACE INTO metadata(key, value) VALUES(?, ?)');
+        db.exec('BEGIN IMMEDIATE');
+        try {
+          if (legacy !== undefined) {
+            const oldDisk = diskSchema.parse(legacy);
+            const insertBlock = db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)');
+            for (const block of oldDisk.blocks) insertBlock.run(block.hash, block.previousHash, JSON.stringify(block));
+            const insertPending = db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
+            oldDisk.pendingTransfers.forEach((tx, index) => insertPending.run(index, transferId(tx), JSON.stringify(tx)));
+          }
+          insertMeta.run('legacy_json_migrated', '1');
+          db.exec('COMMIT');
+        } catch (error) { db.exec('ROLLBACK'); throw error; }
+      }
+
+      const ledger = new Ledger(file, db, pow);
+      for (const row of db.prepare('SELECT block_json FROM blocks ORDER BY sequence').iterate() as Iterable<{ block_json: string }>) {
+        const raw = blockSchema.parse(JSON.parse(row.block_json));
+        if (ledger.nodes.has(raw.hash)) throw new Error('Duplicate persisted block');
+        const node = await ledger.validate(raw);
+        ledger.nodes.set(node.block.hash, node);
+        if (node.work > ledger.tipWork()) ledger.tip = node.block.hash;
+      }
+      let state = ledger.tipState();
+      const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transferId)));
+      for (const row of db.prepare('SELECT id, transaction_json FROM pending_transfers ORDER BY position').iterate() as Iterable<{ id: string; transaction_json: string }>) {
+        const tx = transferSchema.parse(JSON.parse(row.transaction_json));
+        const id = transferId(tx);
+        if (row.id !== id || seen.has(id)) throw new Error('Duplicate or mismatched persisted transfer');
+        state = applyTransferOverlay(state, tx, ledger.height() + 1);
+        seen.add(id);
+        ledger.pending.push(tx);
+      }
+      ledger.pendingState = state;
+      return ledger;
+    } catch (error) {
+      db.close();
+      throw error;
     }
-    const state = structuredClone(ledger.tipState());
-    const seen = new Set(ledger.canonical().flatMap(b => b.transactions.map(transferId)));
-    for (const tx of disk.pendingTransfers) {
-      const id = transferId(tx);
-      if (seen.has(id)) throw new Error('Duplicate persisted transfer');
-      applyTransfer(state, tx, ledger.height() + 1);
-      seen.add(id);
-    }
-    ledger.pending = disk.pendingTransfers;
-    return ledger;
   }
 
   get size(): number { return this.nodes.size; }
-  get allBlocks(): TestnetBlock[] { return this.inventory(0, MAX_BLOCKS); }
+  get allBlocks(): TestnetBlock[] {
+    const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence').all() as { block_json: string }[];
+    return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
+  }
+  close(): void {
+    if (this.closed) return;
+    this.closed = true;
+    this.db.close();
+  }
   block(hash: string): TestnetBlock | undefined {
     const block = this.nodes.get(hash)?.block;
     return block ? structuredClone(block) : undefined;
@@ -91,10 +158,12 @@ export class Ledger {
    * HTTP callers should request limit=32 and return next=offset+length or null.
    */
   inventory(offset: number, limit: number): TestnetBlock[] {
-    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > MAX_BLOCKS) {
+    if (!Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(limit) || limit < 1 || limit > 1000) {
       throw new Error('Invalid inventory range');
     }
-    return structuredClone(Array.from(this.nodes.values()).slice(offset, offset + limit).map(n => n.block));
+    const rows = this.db.prepare('SELECT block_json FROM blocks ORDER BY sequence LIMIT ? OFFSET ?')
+      .all(limit, offset) as { block_json: string }[];
+    return rows.map(row => blockSchema.parse(JSON.parse(row.block_json)));
   }
 
   async submit(raw: unknown): Promise<{ accepted: boolean; reorg: boolean }> {
@@ -102,20 +171,21 @@ export class Ledger {
     return this.serialize(async () => {
       const parsed = blockSchema.parse(input);
       if (this.nodes.has(parsed.hash)) return { accepted: false, reorg: false };
-      if (this.size >= MAX_BLOCKS) throw new Error('Experimental testnet limit: 10000 stored blocks');
       const node = await this.validate(parsed);
       const wins = node.work > this.tipWork();
       const oldChain = this.canonical();
       const chain = wins ? [...this.branch(node.block.previousHash), node.block] : oldChain;
       const hashes = new Set(chain.map(b => b.hash));
       const disconnected = wins ? oldChain.filter(b => !hashes.has(b.hash)) : [];
-      const pending = wins ? this.filterPending(node.state, chain,
-        [...disconnected.flatMap(b => b.transactions), ...this.pending]) : this.pending;
+      const filtered = wins ? this.filterPending(node.state, chain,
+        [...disconnected.flatMap(b => b.transactions), ...this.pending]) : undefined;
+      const pending = filtered?.pending ?? this.pending;
       // Nothing visible changes until the atomic replacement succeeds.
-      await this.persist([...Array.from(this.nodes.values(), n => n.block), node.block], pending);
+      await this.persist(node.block, pending);
       this.nodes.set(node.block.hash, node);
       if (wins) this.tip = node.block.hash;
       this.pending = pending;
+      if (filtered) this.pendingState = filtered.state;
       return { accepted: true, reorg: disconnected.length > 0 };
     });
   }
@@ -127,12 +197,11 @@ export class Ledger {
       const id = transferId(tx);
       if (this.pending.some(t => transferId(t) === id)) return id;
       if (this.pending.length >= MAX_PENDING) throw new Error('Testnet mempool limit: 256 transfers');
-      const state = structuredClone(this.tipState());
-      for (const pending of this.pending) applyTransfer(state, pending, this.height() + 1);
-      applyTransfer(state, tx, this.height() + 1); // No speculative issuance.
+      const nextPendingState = applyTransferOverlay(this.pendingState, tx, this.height() + 1);
       const pending = [...this.pending, tx];
-      await this.persist(Array.from(this.nodes.values(), n => n.block), pending);
+      await this.persist(undefined, pending);
       this.pending = pending;
+      this.pendingState = nextPendingState;
       return id;
     });
   }
@@ -152,7 +221,7 @@ export class Ledger {
   }
 
   private height(): number { return this.nodes.get(this.tip)?.block.height ?? 0; }
-  private tipState(): NftState { return this.nodes.get(this.tip)?.state ?? {}; }
+  private tipState(): NftStateView { return this.nodes.get(this.tip)?.state ?? {}; }
   private tipWork(): bigint { return this.nodes.get(this.tip)?.work ?? 0n; }
   private branch(hash: string): TestnetBlock[] {
     const result: TestnetBlock[] = [];
@@ -171,19 +240,20 @@ export class Ledger {
     const { block, state } = await validateBlock(raw, ancestors, parent?.state ?? {}, this.pow);
     return { block, state, work: (parent?.work ?? 0n) + workFor(block.target) };
   }
-  private filterPending(parent: NftState, chain: TestnetBlock[], candidates: NftTransfer[]): NftTransfer[] {
-    const state = structuredClone(parent);
+  private filterPending(parent: NftStateView, chain: TestnetBlock[], candidates: NftTransfer[]):
+    { pending: NftTransfer[]; state: NftStateView } {
+    let state = parent;
     const seen = new Set(chain.flatMap(b => b.transactions.map(transferId)));
     const pending: NftTransfer[] = [];
     for (const tx of candidates) {
       if (pending.length === MAX_PENDING) break;
       const id = transferId(tx);
       if (seen.has(id)) continue;
-      try { applyTransfer(state, tx, chain.length + 1); }
+      try { state = applyTransferOverlay(state, tx, chain.length + 1); }
       catch { continue; } // Disconnected issuance, immature coins and conflicts are not replayable.
       seen.add(id); pending.push(tx);
     }
-    return pending;
+    return { pending, state };
   }
   private serialize<T>(operation: () => Promise<T>): Promise<T> {
     if (this.queued >= 256) return Promise.reject(new Error('Testnet mutation queue full'));
@@ -192,25 +262,18 @@ export class Ledger {
     this.queue = result.catch(() => undefined).finally(() => { this.queued--; });
     return result;
   }
-  private async persist(blocks: TestnetBlock[], pendingTransfers: NftTransfer[]): Promise<void> {
-    const data = JSON.stringify({ version: 1, network: NETWORK, chainId: CHAIN_ID, blocks, pendingTransfers });
-    if (Buffer.byteLength(data) > MAX_FILE) throw new Error('Testnet ledger file exceeds 128 MiB');
-    const temp = `${this.file}.${randomUUID()}.tmp`;
+  private async persist(block: TestnetBlock | undefined, pendingTransfers: NftTransfer[]): Promise<void> {
+    this.db.exec('BEGIN IMMEDIATE');
     try {
-      const handle = await open(temp, 'wx', 0o600);
-      try { await handle.writeFile(data); await handle.sync(); }
-      finally { await handle.close(); }
-      // Windows scanners can briefly hold the destination open without delete sharing.
-      // Retry only transient sharing failures; never unlink the old durable ledger.
-      for (let attempt = 0; ; attempt++) {
-        try { await rename(temp, this.file); break; }
-        catch (error) {
-          const code = (error as NodeJS.ErrnoException).code;
-          if (process.platform !== 'win32' || attempt >= 5 ||
-            !['EPERM', 'EACCES', 'EBUSY'].includes(code ?? '')) throw error;
-          await delay(10 * 2 ** attempt);
-        }
-      }
-    } finally { await unlink(temp).catch(() => undefined); }
+      if (block) this.db.prepare('INSERT INTO blocks(hash, previous_hash, block_json) VALUES(?, ?, ?)')
+        .run(block.hash, block.previousHash, JSON.stringify(block));
+      this.db.exec('DELETE FROM pending_transfers');
+      const insertPending = this.db.prepare('INSERT INTO pending_transfers(position, id, transaction_json) VALUES(?, ?, ?)');
+      pendingTransfers.forEach((tx, index) => insertPending.run(index, transferId(tx), JSON.stringify(tx)));
+      this.db.exec('COMMIT');
+    } catch (error) {
+      this.db.exec('ROLLBACK');
+      throw error;
+    }
   }
 }

server/testnet/miner.ts

@@ -3,7 +3,7 @@ import { resolve } from 'node:path';
 import { stdin, stdout } from 'node:process';
 import { randomInt } from 'node:crypto';
 import { decryptWallet } from '../../shared/wallet-backup.ts';
-import { blockClaimMessage, NETWORK, type MiningTemplate, type TestnetBlock } from '../../shared/testnet.ts';
+import { blockClaimMessage, CONSENSUS, NETWORK, type MiningTemplate, type TestnetBlock } from '../../shared/testnet.ts';
 import { CHAIN_ID, sha256, miningBlob, mintId, blockSchema } from './consensus.ts';
 import { RandomX } from './randomx.ts';
 
@@ -83,7 +83,7 @@ try {
         header.nonce += 16;
         stdout.write(`\r${(attempts / ((Date.now()-started)/1000)).toFixed(1)} H/s · searching block #${header.height}   `);
         const status = await api('/p2p/status');
-        if (status.tip !== header.previousHash || Date.now()/1000 - header.createdAt > 60) { console.log('\nRefreshing stale work'); break; }
+        if (status.tip !== header.previousHash || Date.now()/1000 - header.createdAt > CONSENSUS.targetSeconds * 2) { console.log('\nRefreshing stale work'); break; }
         // Explicit idle interval; native miner never starts with the web page.
         await new Promise(resolve => setTimeout(resolve,100));
       }

server/testnet/peers.ts

@@ -24,9 +24,9 @@ function publicAddress(ip: string): boolean {
   return family === 6 && globalV6.check(ip, 'ipv6') && !excluded.check(ip, 'ipv6');
 }
 const remoteStatus = z.object({ network: z.literal(NETWORK), chainId: z.literal(CHAIN_ID),
-  size: z.number().int().min(0).max(10_000), tip: z.string().regex(/^[a-f0-9]{64}$/).optional() }).strict();
+  size: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), tip: z.string().regex(/^[a-f0-9]{64}$/).optional() }).strict();
 const pageSchema = z.object({ blocks: z.array(blockSchema).max(32),
-  next: z.number().int().min(0).max(10_000).nullable() }).strict();
+  next: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER).nullable() }).strict();
 const poolSchema = z.object({ transactions: z.array(transferSchema).max(256) }).strict();
 type Peer = { url: string; connected: boolean; error?: string; offset: number; busy: boolean };
 
@@ -121,8 +121,8 @@ export class PeerNetwork {
     if (peer.offset < status.size) {
       const page = pageSchema.parse(await this.request(peer, `/p2p/blocks?offset=${peer.offset}`));
       const end = peer.offset + page.blocks.length;
-      if (!page.blocks.length || end > 10_000 ||
-        (page.next !== null && (page.next !== end || end >= 10_000)) ||
+      if (!page.blocks.length || !Number.isSafeInteger(end) ||
+        (page.next !== null && page.next !== end) ||
         (page.next === null && end < status.size)) throw new Error('Invalid peer pagination');
       for (const block of page.blocks) {
         if (this.closed) throw new Error('Peer network closed');

server/wiki.ts

@@ -74,7 +74,7 @@ export class WikipediaArticleSource {
             generator: 'random', grnnamespace: '0', grnlimit: '10', prop: 'extracts|pageimages|info|categories',
             exintro: '1', explaintext: '1', exchars: '900', piprop: 'thumbnail', pithumbsize: '800', cllimit: '20' }).toString();
           const response = await this.fetcher(url, { signal: AbortSignal.timeout(3500), redirect: 'error',
-            headers: { User-Agent: 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
+            headers: { 'User-Agent': 'Wikideck/0.1 (experimental Wikipedia article cards)', Accept: 'application/json' } });
           const data = responseSchema.parse(await readJson(response));
           for (const rawPage of data.query.pages) {
             const parsedPage = pageSchema.safeParse(rawPage);