More scalable data storage

tests/api.test.ts

@@ -7,6 +7,7 @@ import { tmpdir } from 'node:os';
 import { delimiter, join, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { generateKeyPairSync, sign } from 'node:crypto';
+import { DatabaseSync } from 'node:sqlite';
 import { setTimeout as delay } from 'node:timers/promises';
 import { RandomX } from '../server/testnet/randomx.ts';
 import { address, CHAIN_ID, GENESIS_HASH, MAX_TARGET, miningBlob, mintId, sha256,
@@ -171,7 +172,15 @@ test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and du
       assert.deepEqual(await json(base, '/api/blocks', 201, block), { accepted: true, reorg: false });
       assert.deepEqual(await json(base, '/api/blocks', 200, block), { accepted: false, reorg: false });
     }
-    const persisted = (dir: string) => readFile(join(dir, 'testnet', 'ledger.json'), 'utf8');
+    const persisted = (dir: string) => {
+      const db = new DatabaseSync(join(dir, 'testnet', 'ledger.sqlite'));
+      try {
+        return JSON.stringify({
+          blocks: db.prepare('SELECT hash, previous_hash, block_json FROM blocks ORDER BY sequence').all(),
+          pending: db.prepare('SELECT position, id, transaction_json FROM pending_transfers ORDER BY position').all(),
+        });
+      } finally { db.close(); }
+    };
 
     try {
       engine = new RandomX({ binary }); // Always the actual reference RandomX engine, never a test double.
@@ -209,7 +218,7 @@ test('native RandomX HTTP: three blocks, mined transfer, peer gossip/sync and du
       for (const path of ['/api/missing', '/p2p/missing', `/api/nfts/${'0'.repeat(64)}`]) {
         await (await request(first.base, path, 404)).text();
       }
-      for (const offset of ['-1', '1.5', '10001', 'nope']) {
+      for (const offset of ['-1', '1.5', '9007199254740992', 'nope']) {
         await (await request(first.base, `/p2p/blocks?offset=${offset}`, 400)).text();
       }
 

tests/testnet-consensus.test.ts

@@ -1,7 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { merkleRoot, sha256, expectedKey, INITIAL_KEY, expectedTarget, MAX_TARGET, CHAIN_ID } from '../server/testnet/consensus.ts';
-import { NETWORK, type TestnetBlock } from '../shared/testnet.ts';
+import { CONSENSUS, NETWORK, type TestnetBlock } from '../shared/testnet.ts';
 
 test('Merkle commitments distinguish padded leaves, ordering, and node/leaf domains',()=>{
   const a=sha256('a'),b=sha256('b'),c=sha256('c');
@@ -20,7 +20,7 @@ test('RandomX keys rotate only after the protocol delay using an ancestor, never
 });
 test('difficulty retargets deterministically at exact windows and is bounded',()=>{
   assert.equal(expectedTarget([]),MAX_TARGET);
-  const blocks=Array.from({length:10},(_,i)=>({target:MAX_TARGET,createdAt:1000+i*30}as TestnetBlock));
+  const blocks=Array.from({length:10},(_,i)=>({target:MAX_TARGET,createdAt:1000+i*CONSENSUS.targetSeconds}as TestnetBlock));
   assert.equal(expectedTarget(blocks.slice(0,9)),MAX_TARGET);
   assert.equal(expectedTarget(blocks),MAX_TARGET);
   const fast=blocks.map((b,i)=>({...b,createdAt:1000+i}));

tests/testnet-ledger.test.ts

@@ -6,13 +6,15 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
 import dns from 'node:dns/promises';
+import { DatabaseSync } from 'node:sqlite';
 import { setTimeout as delay } from 'node:timers/promises';
 import { Ledger } from '../server/testnet/ledger.ts';
 import { PeerNetwork } from '../server/testnet/peers.ts';
 import { CHAIN_ID, GENESIS_HASH, GENESIS_TIME, MAX_TARGET, address, makeTemplate, miningBlob,
   mintId, sha256, stateRoot, transferId, transition, workFor } from '../server/testnet/consensus.ts';
 import { NETWORK, CONSENSUS, blockClaimMessage, nftTransferMessage, testnetOwnerOf,
-  type MiningTemplate, type NftState, type NftTransfer, type TestnetBlock } from '../shared/testnet.ts';
+  type MiningTemplate, type NftState, type NftStateLayer, type NftStateView,
+  type NftTransfer, type TestnetBlock } from '../shared/testnet.ts';
 import { deriveStats } from '../shared/enrichment.ts';
 import type { PowEngine } from '../server/testnet/randomx.ts';
 
@@ -47,7 +49,7 @@ async function mine(template: MiningTemplate, miner: Identity = alice): Promise<
   }
   throw new Error('Test nonce search exhausted');
 }
-type Branch = { blocks: TestnetBlock[]; state: NftState };
+type Branch = { blocks: TestnetBlock[]; state: NftStateView };
 function branch(): Branch { return { blocks: [], state: {} }; }
 function articleFor(_ancestors: TestnetBlock[]) {
   const pageId = 50_000;
@@ -77,10 +79,10 @@ function transfer(cardHash: string, from: Identity = alice, to: Identity = bob,
 }
 async function fixture(t: { after: (fn: () => Promise<void>) => void }) {
   const dir = await mkdtemp(join(tmpdir(), 'wikideck-testnet-'));
-  t.after(() => rm(dir, { recursive: true, force: true }));
   const engine = new FakePow();
   const ledger = await Ledger.open(dir, engine);
-  return { dir, ledger, engine, file: join(dir, 'testnet', 'ledger.json') };
+  t.after(async () => { ledger.close(); engine.close(); await rm(dir, { recursive: true, force: true }); });
+  return { dir, ledger, engine, file: join(dir, 'testnet', 'ledger.sqlite') };
 }
 
 test('testnet genesis is empty, manifest-bound, isolated and defensively copied', async t => {
@@ -99,10 +101,15 @@ test('testnet genesis is empty, manifest-bound, isolated and defensively copied'
   assert.deepEqual(initial.pendingTransfers, []);
   assert.equal(stateRoot({}), sha256(`${NETWORK}:empty`));
   await assert.rejects(readFile(join(dir, 'chain.json')), /ENOENT/);
-  assert.equal(JSON.parse(await readFile(file, 'utf8')).version, 1);
-  if (process.platform !== 'win32') assert.equal((await stat(file)).mode & 0o777, 0o600);
+  assert.equal((await stat(file)).isFile(), true);
+  const database = new DatabaseSync(file);
+  assert.equal((database.prepare('SELECT COUNT(*) AS count FROM blocks').get() as { count: number }).count, 0);
+  database.close();
   const candidate = await mine(ledger.template(alice.owner, alice.publicKey, articleFor([]), statsSnapshot()));
   assert.deepEqual(await ledger.submit(candidate), { accepted: true, reorg: false });
+  const persisted = new DatabaseSync(file);
+  assert.equal((persisted.prepare('SELECT COUNT(*) AS count FROM blocks').get() as { count: number }).count, 1);
+  persisted.close();
   candidate.article.title = 'caller mutation';
   const copy = ledger.block(candidate.hash)!;
   copy.article.title = 'getter mutation';
@@ -114,8 +121,9 @@ test('testnet genesis is empty, manifest-bound, isolated and defensively copied'
 
 test('DAG retains equal-work branches, serializes duplicates and selects strictly greater cumulative work', async t => {
   const { ledger, dir } = await fixture(t);
+  t.mock.method(Date, 'now', () => (GENESIS_TIME + CONSENSUS.targetSeconds * 20) * 1000);
   const slow = branch(), fast = branch();
-  for (let i = 0; i < 12; i++) await ledger.submit(await extend(slow, alice, [], 30));
+  for (let i = 0; i < 12; i++) await ledger.submit(await extend(slow, alice, [], CONSENSUS.targetSeconds));
   for (let i = 0; i < 10; i++) {
     const result = await ledger.submit(await extend(fast, bob, [], 1));
     assert.equal(result.reorg, false);
@@ -136,6 +144,7 @@ test('DAG retains equal-work branches, serializes duplicates and selects strictl
   for (const b of stored) { assert.ok(seen.has(b.previousHash)); seen.add(b.hash); }
   assert.deepEqual(ledger.inventory(10, 3), stored.slice(10, 13));
   const restart = await Ledger.open(dir, new FakePow());
+  t.after(async () => { restart.close(); });
   assert.deepEqual(restart.snapshot(), ledger.snapshot());
   assert.deepEqual(restart.allBlocks, stored);
 });
@@ -148,6 +157,7 @@ test('equal-work ties preserve the current tip across restart and only extension
   assert.deepEqual(await ledger.submit(b1), { accepted: true, reorg: false });
   assert.equal(ledger.snapshot().tip, a1.hash);
   const restart = await Ledger.open(dir, new FakePow());
+  t.after(async () => { restart.close(); });
   assert.equal(restart.snapshot().tip, a1.hash);
   const b2 = await extend(b, bob);
   assert.deepEqual(await restart.submit(b2), { accepted: true, reorg: true });
@@ -169,6 +179,7 @@ test('NFT maturity, ordered mempool, block commitments and replay protection', a
   assert.deepEqual(ledger.snapshot().pendingTransfers, [tx, tx2]);
   assert.equal(testnetOwnerOf(ledger.snapshot(), minted.tokenId), alice.owner); // Unconfirmed is not ownership.
   const restarted = await Ledger.open(dir, new FakePow());
+  t.after(async () => { restarted.close(); });
   assert.deepEqual(restarted.snapshot().pendingTransfers, [tx, tx2]);
   const block = await mine(restarted.template(alice.owner, alice.publicKey,
     articleFor(restarted.snapshot().blocks), statsSnapshot()));
@@ -203,7 +214,9 @@ test('reorg rolls back NFT ownership and requeues only still-valid disconnected
   assert.deepEqual(snapshot.transfers, []);
   assert.deepEqual(snapshot.pendingTransfers, [tx, tx2]); // Orphan issuance is discarded.
   assert.ok(ledger.block(committed.hash));
-  assert.deepEqual((await Ledger.open(dir, new FakePow())).snapshot(), snapshot);
+  const restored = await Ledger.open(dir, new FakePow());
+  t.after(async () => { restored.close(); });
+  assert.deepEqual(restored.snapshot(), snapshot);
   const next = await mine(ledger.template(bob.owner, bob.publicKey,
     articleFor(ledger.snapshot().blocks), statsSnapshot()), bob);
   await ledger.submit(next);
@@ -252,53 +265,88 @@ test('invalid parent, manifest, proof, article/stat snapshots, and commitments l
   assert.equal(ledger.size, 2);
 });
 
-test('startup rechecks every proof, rejects tampering and mismatched manifests without resetting disk', async t => {
+test('block state is stored as a one-entry delta that shares its parent state layer', async () => {
+  const parent: NftState = Object.assign(Object.create(null) as NftState, {
+    ['a'.repeat(64)]: { owner: alice.owner, sequence: 0, mintedHeight: 1, pageId: 50_000 },
+  });
+  const chain: Branch = { blocks: [], state: parent };
+  await extend(chain);
+  const layer = chain.state as NftStateLayer;
+  assert.equal(layer.kind, 'layer');
+  assert.equal(layer.parent, parent);
+  assert.equal(Object.keys(layer.changes).length, 1);
+});
+
+test('SQLite startup rechecks every proof and fails closed on database corruption or manifest mismatch', async t => {
   const { dir, ledger, file } = await fixture(t);
   const chain = branch();
   await ledger.submit(await extend(chain));
   await ledger.submit(await extend(chain));
   const checking = new FakePow();
-  await Ledger.open(dir, checking);
+  const verified = await Ledger.open(dir, checking);
+  t.after(async () => { verified.close(); });
   assert.equal(checking.calls, 2);
-  const original = await readFile(file, 'utf8');
-  const disk = JSON.parse(original);
-  for (const changed of [ { ...disk, chainId: 'ff'.repeat(32) }, { ...disk, network: 'wrong' },
-    { ...disk, version: 2 }, { ...disk, blocks: [...disk.blocks].reverse() },
-    { ...disk, blocks: [disk.blocks[0], disk.blocks[0]] },
-    { ...disk, blocks: [{ ...disk.blocks[0], stateRoot: '00'.repeat(32) }] },
-    { ...disk, pendingTransfers: [transfer(chain.blocks[0].tokenId, bob)] } ]) {
-    const text = JSON.stringify(changed);
-    await writeFile(file, text);
-    await assert.rejects(Ledger.open(dir, new FakePow()));
-    assert.equal(await readFile(file, 'utf8'), text);
-  }
-  await writeFile(file, original);
-  await assert.rejects(Ledger.open(dir, { hash: async () => 'ff'.repeat(32), close() {} }), /RandomX proof/);
-  await writeFile(file, '{truncated');
+  const originalDb = new DatabaseSync(file);
+  const row = originalDb.prepare('SELECT hash, block_json FROM blocks ORDER BY sequence LIMIT 1').get() as { hash: string; block_json: string };
+  const originalBlock = JSON.parse(row.block_json) as TestnetBlock;
+  originalDb.close();
+  verified.close();
+  ledger.close();
+
+  const malformed = new DatabaseSync(file);
+  malformed.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run('{truncated', row.hash);
+  malformed.close();
   await assert.rejects(Ledger.open(dir, new FakePow()));
+  const restoreMalformed = new DatabaseSync(file);
+  restoreMalformed.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run(row.block_json, row.hash);
+  restoreMalformed.close();
+
+  const tampered = new DatabaseSync(file);
+  tampered.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?')
+    .run(JSON.stringify({ ...originalBlock, stateRoot: '00'.repeat(32) }), row.hash);
+  tampered.close();
+  await assert.rejects(Ledger.open(dir, new FakePow()), /state commitment/);
+  const restoreTampered = new DatabaseSync(file);
+  restoreTampered.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run(row.block_json, row.hash);
+  restoreTampered.prepare("UPDATE metadata SET value = 'wrong' WHERE key = 'network'").run();
+  restoreTampered.close();
+  await assert.rejects(Ledger.open(dir, new FakePow()), /mismatch/);
+  const restoreManifest = new DatabaseSync(file);
+  restoreManifest.prepare('UPDATE metadata SET value = ? WHERE key = ?').run(NETWORK, 'network');
+  restoreManifest.close();
+  await assert.rejects(Ledger.open(dir, { hash: async () => 'ff'.repeat(32), close() {} }), /RandomX proof/);
 });
 
-test('failed atomic replacement never commits memory, and queued mutations recover', async t => {
+test('failed SQLite transaction never commits memory, and later mutations recover', async t => {
   const { ledger, file } = await fixture(t);
-  const saved = `${file}.saved`;
-  await rename(file, saved);
-  await mkdir(file);
+  const database = new DatabaseSync(file);
+  database.exec("CREATE TRIGGER reject_block BEFORE INSERT ON blocks BEGIN SELECT RAISE(ABORT, 'test persistence failure'); END;");
+  database.close();
   const valid = await extend(branch());
   await assert.rejects(ledger.submit(valid));
   assert.equal(ledger.size, 0);
   assert.equal(ledger.snapshot().tip, GENESIS_HASH);
-  await rm(file, { recursive: true });
-  await rename(saved, file);
+  const removeTrigger = new DatabaseSync(file);
+  removeTrigger.exec('DROP TRIGGER reject_block');
+  removeTrigger.close();
   await ledger.submit(valid);
   assert.equal(ledger.size, 1);
 });
 
-test('bounded persistence rejects oversized files before reading or replaying proofs', async t => {
-  const { dir, file } = await fixture(t);
-  await truncate(file, 128 * 1024 * 1024 + 1);
-  const engine = new FakePow();
-  await assert.rejects(Ledger.open(dir, engine), /128 MiB/);
-  assert.equal(engine.calls, 0);
+test('legacy JSON ledger imports once into SQLite without altering its source file', async t => {
+  const dir = await mkdtemp(join(tmpdir(), 'wikideck-legacy-ledger-'));
+  const block = await extend(branch());
+  const original = { version: 1, network: NETWORK, chainId: CHAIN_ID, blocks: [block], pendingTransfers: [] };
+  const legacyFile = join(dir, 'testnet', 'ledger.json');
+  await mkdir(join(dir, 'testnet'), { recursive: true });
+  await writeFile(legacyFile, JSON.stringify(original));
+  const ledger = await Ledger.open(dir, new FakePow());
+  t.after(async () => { ledger.close(); await rm(dir, { recursive: true, force: true }); });
+  assert.deepEqual(ledger.snapshot().blocks, [block]);
+  assert.deepEqual(JSON.parse(await readFile(legacyFile, 'utf8')), original);
+  const database = new DatabaseSync(join(dir, 'testnet', 'ledger.sqlite'));
+  assert.equal((database.prepare("SELECT value FROM metadata WHERE key = 'legacy_json_migrated'").get() as { value: string }).value, '1');
+  database.close();
 });
 
 test('mempool is ordered, capped at 256, and templates include at most 64 without new issuance', async t => {
@@ -321,6 +369,7 @@ test('mempool is ordered, capped at 256, and templates include at most 64 withou
   assert.deepEqual(ledger.snapshot().pendingTransfers, transactions.slice(64));
   await assert.rejects(ledger.submitTransfer(transfer(block.tokenId)), /immature/);
   const restarted = await Ledger.open(dir, new FakePow());
+  t.after(async () => { restarted.close(); });
   assert.deepEqual(restarted.snapshot(), ledger.snapshot());
 });
 
@@ -374,6 +423,22 @@ test('peer configuration rejects unsafe origins; explicit private permission ena
   assert.equal(ledger.snapshot().peers, 0);
 });
 
+
+test('unknown SQLite files are rejected without adding or replacing tables', async t => {
+  const dir = await mkdtemp(join(tmpdir(), 'wikideck-foreign-sqlite-'));
+  const file = join(dir, 'testnet', 'ledger.sqlite');
+  await mkdir(join(dir, 'testnet'), { recursive: true });
+  const foreign = new DatabaseSync(file);
+  foreign.exec('CREATE TABLE operator_data(value TEXT NOT NULL); INSERT INTO operator_data VALUES(\'keep\');');
+  foreign.close();
+  t.after(async () => { await rm(dir, { recursive: true, force: true }); });
+  await assert.rejects(Ledger.open(dir, new FakePow()), /metadata missing/);
+  const verify = new DatabaseSync(file);
+  assert.deepEqual((verify.prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name").all() as { name: string }[])
+    .map(row => row.name), ['operator_data']);
+  assert.equal((verify.prepare('SELECT value FROM operator_data').get() as { value: string }).value, 'keep');
+  verify.close();
+});
 test('peers verify DNS, manifests, pagination, payloads and consensus; redirects are not followed', async t => {
   const { ledger } = await fixture(t);
   const chain = branch();

tests/wallet-backup.test.ts

@@ -54,7 +54,7 @@ test('Node WebCrypto backup roundtrip preserves SPKI SHA256 identity and signing
     key: createPublicKey({ key: spki, format: 'der', type: 'spki' }), dsaEncoding: 'ieee-p1363',
   }, Buffer.from(signature, 'base64')), true);
   assert.equal(backup.iterations, 600000);
-  assert.equal(backup.network, 'wikideck-randomx-testnet-4');
+  assert.equal(backup.network, 'wikideck-randomx-testnet-5');
   assert.equal(Buffer.from(backup.salt, 'base64').length, 16);
   assert.equal(Buffer.from(backup.iv, 'base64').length, 12);
   assert.ok(Buffer.byteLength(JSON.stringify(backup)) < 32 * 1024);