WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

More scalable data storage

Browse Source
This commit is contained in:
eric
2026-10-19 20:17:10 -04:00
parent 0b9061372b
commit baad477ca5
16 changed files with 346 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/testnet-ledger.test.ts
+102 -37
View File
@@ -6,13 +6,15 @@ import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
import dns from 'node:dns/promises';
import { DatabaseSync } from 'node:sqlite';
import { setTimeout as delay } from 'node:timers/promises';
import { Ledger } from '../server/testnet/ledger.ts';
import { PeerNetwork } from '../server/testnet/peers.ts';
import { CHAIN_ID, GENESIS_HASH, GENESIS_TIME, MAX_TARGET, address, makeTemplate, miningBlob,
mintId, sha256, stateRoot, transferId, transition, workFor } from '../server/testnet/consensus.ts';
import { NETWORK, CONSENSUS, blockClaimMessage, nftTransferMessage, testnetOwnerOf,
type MiningTemplate, type NftState, type NftTransfer, type TestnetBlock } from '../shared/testnet.ts';
type MiningTemplate, type NftState, type NftStateLayer, type NftStateView,
type NftTransfer, type TestnetBlock } from '../shared/testnet.ts';
import { deriveStats } from '../shared/enrichment.ts';
import type { PowEngine } from '../server/testnet/randomx.ts';
@@ -47,7 +49,7 @@ async function mine(template: MiningTemplate, miner: Identity = alice): Promise<
}
throw new Error('Test nonce search exhausted');
}
type Branch = { blocks: TestnetBlock[]; state: NftState };
type Branch = { blocks: TestnetBlock[]; state: NftStateView };
function branch(): Branch { return { blocks: [], state: {} }; }
function articleFor(_ancestors: TestnetBlock[]) {
const pageId = 50_000;
@@ -77,10 +79,10 @@ function transfer(cardHash: string, from: Identity = alice, to: Identity = bob,
}
async function fixture(t: { after: (fn: () => Promise<void>) => void }) {
const dir = await mkdtemp(join(tmpdir(), 'wikideck-testnet-'));
t.after(() => rm(dir, { recursive: true, force: true }));
const engine = new FakePow();
const ledger = await Ledger.open(dir, engine);
return { dir, ledger, engine, file: join(dir, 'testnet', 'ledger.json') };
t.after(async () => { ledger.close(); engine.close(); await rm(dir, { recursive: true, force: true }); });
return { dir, ledger, engine, file: join(dir, 'testnet', 'ledger.sqlite') };
}
test('testnet genesis is empty, manifest-bound, isolated and defensively copied', async t => {
@@ -99,10 +101,15 @@ test('testnet genesis is empty, manifest-bound, isolated and defensively copied'
assert.deepEqual(initial.pendingTransfers, []);
assert.equal(stateRoot({}), sha256(`${NETWORK}:empty`));
await assert.rejects(readFile(join(dir, 'chain.json')), /ENOENT/);
assert.equal(JSON.parse(await readFile(file, 'utf8')).version, 1);
if (process.platform !== 'win32') assert.equal((await stat(file)).mode & 0o777, 0o600);
assert.equal((await stat(file)).isFile(), true);
const database = new DatabaseSync(file);
assert.equal((database.prepare('SELECT COUNT(*) AS count FROM blocks').get() as { count: number }).count, 0);
database.close();
const candidate = await mine(ledger.template(alice.owner, alice.publicKey, articleFor([]), statsSnapshot()));
assert.deepEqual(await ledger.submit(candidate), { accepted: true, reorg: false });
const persisted = new DatabaseSync(file);
assert.equal((persisted.prepare('SELECT COUNT(*) AS count FROM blocks').get() as { count: number }).count, 1);
persisted.close();
candidate.article.title = 'caller mutation';
const copy = ledger.block(candidate.hash)!;
copy.article.title = 'getter mutation';
@@ -114,8 +121,9 @@ test('testnet genesis is empty, manifest-bound, isolated and defensively copied'
test('DAG retains equal-work branches, serializes duplicates and selects strictly greater cumulative work', async t => {
const { ledger, dir } = await fixture(t);
t.mock.method(Date, 'now', () => (GENESIS_TIME + CONSENSUS.targetSeconds * 20) * 1000);
const slow = branch(), fast = branch();
for (let i = 0; i < 12; i++) await ledger.submit(await extend(slow, alice, [], 30));
for (let i = 0; i < 12; i++) await ledger.submit(await extend(slow, alice, [], CONSENSUS.targetSeconds));
for (let i = 0; i < 10; i++) {
const result = await ledger.submit(await extend(fast, bob, [], 1));
assert.equal(result.reorg, false);
@@ -136,6 +144,7 @@ test('DAG retains equal-work branches, serializes duplicates and selects strictl
for (const b of stored) { assert.ok(seen.has(b.previousHash)); seen.add(b.hash); }
assert.deepEqual(ledger.inventory(10, 3), stored.slice(10, 13));
const restart = await Ledger.open(dir, new FakePow());
t.after(async () => { restart.close(); });
assert.deepEqual(restart.snapshot(), ledger.snapshot());
assert.deepEqual(restart.allBlocks, stored);
});
@@ -148,6 +157,7 @@ test('equal-work ties preserve the current tip across restart and only extension
assert.deepEqual(await ledger.submit(b1), { accepted: true, reorg: false });
assert.equal(ledger.snapshot().tip, a1.hash);
const restart = await Ledger.open(dir, new FakePow());
t.after(async () => { restart.close(); });
assert.equal(restart.snapshot().tip, a1.hash);
const b2 = await extend(b, bob);
assert.deepEqual(await restart.submit(b2), { accepted: true, reorg: true });
@@ -169,6 +179,7 @@ test('NFT maturity, ordered mempool, block commitments and replay protection', a
assert.deepEqual(ledger.snapshot().pendingTransfers, [tx, tx2]);
assert.equal(testnetOwnerOf(ledger.snapshot(), minted.tokenId), alice.owner); // Unconfirmed is not ownership.
const restarted = await Ledger.open(dir, new FakePow());
t.after(async () => { restarted.close(); });
assert.deepEqual(restarted.snapshot().pendingTransfers, [tx, tx2]);
const block = await mine(restarted.template(alice.owner, alice.publicKey,
articleFor(restarted.snapshot().blocks), statsSnapshot()));
@@ -203,7 +214,9 @@ test('reorg rolls back NFT ownership and requeues only still-valid disconnected
assert.deepEqual(snapshot.transfers, []);
assert.deepEqual(snapshot.pendingTransfers, [tx, tx2]); // Orphan issuance is discarded.
assert.ok(ledger.block(committed.hash));
assert.deepEqual((await Ledger.open(dir, new FakePow())).snapshot(), snapshot);
const restored = await Ledger.open(dir, new FakePow());
t.after(async () => { restored.close(); });
assert.deepEqual(restored.snapshot(), snapshot);
const next = await mine(ledger.template(bob.owner, bob.publicKey,
articleFor(ledger.snapshot().blocks), statsSnapshot()), bob);
await ledger.submit(next);
@@ -252,53 +265,88 @@ test('invalid parent, manifest, proof, article/stat snapshots, and commitments l
assert.equal(ledger.size, 2);
});
test('startup rechecks every proof, rejects tampering and mismatched manifests without resetting disk', async t => {
test('block state is stored as a one-entry delta that shares its parent state layer', async () => {
const parent: NftState = Object.assign(Object.create(null) as NftState, {
['a'.repeat(64)]: { owner: alice.owner, sequence: 0, mintedHeight: 1, pageId: 50_000 },
});
const chain: Branch = { blocks: [], state: parent };
await extend(chain);
const layer = chain.state as NftStateLayer;
assert.equal(layer.kind, 'layer');
assert.equal(layer.parent, parent);
assert.equal(Object.keys(layer.changes).length, 1);
});
test('SQLite startup rechecks every proof and fails closed on database corruption or manifest mismatch', async t => {
const { dir, ledger, file } = await fixture(t);
const chain = branch();
await ledger.submit(await extend(chain));
await ledger.submit(await extend(chain));
const checking = new FakePow();
await Ledger.open(dir, checking);
const verified = await Ledger.open(dir, checking);
t.after(async () => { verified.close(); });
assert.equal(checking.calls, 2);
const original = await readFile(file, 'utf8');
const disk = JSON.parse(original);
for (const changed of [ { ...disk, chainId: 'ff'.repeat(32) }, { ...disk, network: 'wrong' },
{ ...disk, version: 2 }, { ...disk, blocks: [...disk.blocks].reverse() },
{ ...disk, blocks: [disk.blocks[0], disk.blocks[0]] },
{ ...disk, blocks: [{ ...disk.blocks[0], stateRoot: '00'.repeat(32) }] },
{ ...disk, pendingTransfers: [transfer(chain.blocks[0].tokenId, bob)] } ]) {
const text = JSON.stringify(changed);
await writeFile(file, text);
await assert.rejects(Ledger.open(dir, new FakePow()));
assert.equal(await readFile(file, 'utf8'), text);
}
await writeFile(file, original);
await assert.rejects(Ledger.open(dir, { hash: async () => 'ff'.repeat(32), close() {} }), /RandomX proof/);
await writeFile(file, '{truncated');
const originalDb = new DatabaseSync(file);
const row = originalDb.prepare('SELECT hash, block_json FROM blocks ORDER BY sequence LIMIT 1').get() as { hash: string; block_json: string };
const originalBlock = JSON.parse(row.block_json) as TestnetBlock;
originalDb.close();
verified.close();
ledger.close();
const malformed = new DatabaseSync(file);
malformed.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run('{truncated', row.hash);
malformed.close();
await assert.rejects(Ledger.open(dir, new FakePow()));
const restoreMalformed = new DatabaseSync(file);
restoreMalformed.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run(row.block_json, row.hash);
restoreMalformed.close();
const tampered = new DatabaseSync(file);
tampered.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?')
.run(JSON.stringify({ ...originalBlock, stateRoot: '00'.repeat(32) }), row.hash);
tampered.close();
await assert.rejects(Ledger.open(dir, new FakePow()), /state commitment/);
const restoreTampered = new DatabaseSync(file);
restoreTampered.prepare('UPDATE blocks SET block_json = ? WHERE hash = ?').run(row.block_json, row.hash);
restoreTampered.prepare("UPDATE metadata SET value = 'wrong' WHERE key = 'network'").run();
restoreTampered.close();
await assert.rejects(Ledger.open(dir, new FakePow()), /mismatch/);
const restoreManifest = new DatabaseSync(file);
restoreManifest.prepare('UPDATE metadata SET value = ? WHERE key = ?').run(NETWORK, 'network');
restoreManifest.close();
await assert.rejects(Ledger.open(dir, { hash: async () => 'ff'.repeat(32), close() {} }), /RandomX proof/);
});
test('failed atomic replacement never commits memory, and queued mutations recover', async t => {
test('failed SQLite transaction never commits memory, and later mutations recover', async t => {
const { ledger, file } = await fixture(t);
const saved = `${file}.saved`;
await rename(file, saved);
await mkdir(file);
const database = new DatabaseSync(file);
database.exec("CREATE TRIGGER reject_block BEFORE INSERT ON blocks BEGIN SELECT RAISE(ABORT, 'test persistence failure'); END;");
database.close();
const valid = await extend(branch());
await assert.rejects(ledger.submit(valid));
assert.equal(ledger.size, 0);
assert.equal(ledger.snapshot().tip, GENESIS_HASH);
await rm(file, { recursive: true });
await rename(saved, file);
const removeTrigger = new DatabaseSync(file);
removeTrigger.exec('DROP TRIGGER reject_block');
removeTrigger.close();
await ledger.submit(valid);
assert.equal(ledger.size, 1);
});
test('bounded persistence rejects oversized files before reading or replaying proofs', async t => {
const { dir, file } = await fixture(t);
await truncate(file, 128 * 1024 * 1024 + 1);
const engine = new FakePow();
await assert.rejects(Ledger.open(dir, engine), /128 MiB/);
assert.equal(engine.calls, 0);
test('legacy JSON ledger imports once into SQLite without altering its source file', async t => {
const dir = await mkdtemp(join(tmpdir(), 'wikideck-legacy-ledger-'));
const block = await extend(branch());
const original = { version: 1, network: NETWORK, chainId: CHAIN_ID, blocks: [block], pendingTransfers: [] };
const legacyFile = join(dir, 'testnet', 'ledger.json');
await mkdir(join(dir, 'testnet'), { recursive: true });
await writeFile(legacyFile, JSON.stringify(original));
const ledger = await Ledger.open(dir, new FakePow());
t.after(async () => { ledger.close(); await rm(dir, { recursive: true, force: true }); });
assert.deepEqual(ledger.snapshot().blocks, [block]);
assert.deepEqual(JSON.parse(await readFile(legacyFile, 'utf8')), original);
const database = new DatabaseSync(join(dir, 'testnet', 'ledger.sqlite'));
assert.equal((database.prepare("SELECT value FROM metadata WHERE key = 'legacy_json_migrated'").get() as { value: string }).value, '1');
database.close();
});
test('mempool is ordered, capped at 256, and templates include at most 64 without new issuance', async t => {
@@ -321,6 +369,7 @@ test('mempool is ordered, capped at 256, and templates include at most 64 withou
assert.deepEqual(ledger.snapshot().pendingTransfers, transactions.slice(64));
await assert.rejects(ledger.submitTransfer(transfer(block.tokenId)), /immature/);
const restarted = await Ledger.open(dir, new FakePow());
t.after(async () => { restarted.close(); });
assert.deepEqual(restarted.snapshot(), ledger.snapshot());
});
@@ -374,6 +423,22 @@ test('peer configuration rejects unsafe origins; explicit private permission ena
assert.equal(ledger.snapshot().peers, 0);
});
test('unknown SQLite files are rejected without adding or replacing tables', async t => {
const dir = await mkdtemp(join(tmpdir(), 'wikideck-foreign-sqlite-'));
const file = join(dir, 'testnet', 'ledger.sqlite');
await mkdir(join(dir, 'testnet'), { recursive: true });
const foreign = new DatabaseSync(file);
foreign.exec('CREATE TABLE operator_data(value TEXT NOT NULL); INSERT INTO operator_data VALUES(\'keep\');');
foreign.close();
t.after(async () => { await rm(dir, { recursive: true, force: true }); });
await assert.rejects(Ledger.open(dir, new FakePow()), /metadata missing/);
const verify = new DatabaseSync(file);
assert.deepEqual((verify.prepare("SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name").all() as { name: string }[])
.map(row => row.name), ['operator_data']);
assert.equal((verify.prepare('SELECT value FROM operator_data').get() as { value: string }).value, 'keep');
verify.close();
});
test('peers verify DNS, manifests, pagination, payloads and consensus; redirects are not followed', async t => {
const { ledger } = await fixture(t);
const chain = branch();