WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Moved canonical transaction, NFT, and mempool fact reads into the reconciliation worker's own read-only SQLite snapshot

Browse Source
This commit is contained in:
eric
2026-10-21 12:18:33 -04:00
parent 9282c0234d
commit 4566271ce3
7 changed files with 245 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark-market-reconciliation.ts
+39 -9
View File
@@ -1,5 +1,5 @@
import { DatabaseSync } from 'node:sqlite';
import { mkdtemp, rm, stat } from 'node:fs/promises';
import { mkdir, mkdtemp, rm, stat } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
@@ -18,20 +18,45 @@ if (!['terminal', 'confirmed-delivery'].includes(scenario)) throw new Error('Sce
const workerEnabled = process.argv[4] === 'worker';
const directory = await mkdtemp(join(tmpdir(), 'wikideck-market-reconciliation-'));
const streamId = randomUUID(), tip = 'a'.repeat(64);
const deliveryRows = new Map<string, { height: number; blockHash: string }>();
const ledgerFile = join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet', 'ledger.sqlite');
const seller = '1'.repeat(64), buyer = '2'.repeat(64);
let market: Market | undefined;
let ledgerDb: DatabaseSync | undefined;
try {
// Indexed-only adapter: the benchmark must not fall back to materializing chain snapshots.
await mkdir(join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet'), { recursive: true });
ledgerDb = new DatabaseSync(ledgerFile);
ledgerDb.exec(`PRAGMA journal_mode=WAL; CREATE TABLE blocks(hash TEXT PRIMARY KEY,block_json TEXT NOT NULL);
CREATE TABLE canonical_blocks(height INTEGER PRIMARY KEY,hash TEXT NOT NULL);
CREATE TABLE canonical_nfts(token TEXT PRIMARY KEY,owner TEXT NOT NULL,sequence INTEGER NOT NULL,minted_height INTEGER NOT NULL,block_hash TEXT NOT NULL);
CREATE TABLE canonical_transactions(id TEXT PRIMARY KEY,height INTEGER NOT NULL,block_hash TEXT NOT NULL);
CREATE TABLE canonical_events(sequence INTEGER PRIMARY KEY,type TEXT NOT NULL);
CREATE TABLE pending_transfers(position INTEGER PRIMARY KEY,id TEXT NOT NULL,transaction_json TEXT NOT NULL);`);
const insertBlock = ledgerDb.prepare('INSERT INTO blocks VALUES(?,?)');
const insertTip = ledgerDb.prepare('INSERT INTO canonical_blocks VALUES(?,?)');
const insertNft = ledgerDb.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)');
const priorHash = 'b'.repeat(64);
for (const [height, hash] of [[1, priorHash], [2, tip]] as const) {
insertBlock.run(hash, JSON.stringify({ height, hash, tokenId: '', owner: seller, transactions: [] }));
insertTip.run(height, hash);
}
const chain = {
snapshot: () => { throw new Error('Benchmark must not request a full chain snapshot'); },
summary: () => ({ tip, revision: 0, height: 2 }),
nft: () => undefined,
canonicalTransaction: (id: string) => deliveryRows.get(id),
pendingTransactions: () => [],
nft: (token: string) => {
const row = ledgerDb!.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.token=?`).get(token);
if (!row) return undefined;
return { block: JSON.parse(row.block_json), owner: row.owner, sequence: row.sequence,
confirmations: 2 - row.minted_height + 1 };
},
canonicalTransaction: (id: string) => ledgerDb!.prepare('SELECT height,block_hash AS blockHash FROM canonical_transactions WHERE id=?').get(id),
pendingTransactions: () => ledgerDb!.prepare('SELECT transaction_json FROM pending_transfers ORDER BY position').all()
.map(row => JSON.parse(row.transaction_json)),
canonicalHead: () => 0,
canonicalEvents: (after: number) => ({ streamId, previousAnchor: GENESIS_HASH, anchor: GENESIS_HASH,
events: [], cursor: after, head: 0, caughtUp: true }),
...(workerEnabled ? { reconciliationDatabasePath: () => ledgerFile } : {}),
submitTransfer: async () => { throw new Error('Benchmark never submits transfers'); },
};
market = await Market.open(directory, chain, { feeBps: 250, confirmations: 2 }, Date.now, { worker: workerEnabled });
@@ -39,6 +64,8 @@ try {
try {
const insertListing = db.prepare('INSERT INTO listings(id,card,status,data) VALUES(?,?,?,?)');
const insertOrder = db.prepare('INSERT INTO orders(id,data) VALUES(?,?)');
const insertNft = ledgerDb?.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)');
const insertTransaction = ledgerDb?.prepare('INSERT INTO canonical_transactions VALUES(?,?,?)');
db.exec('BEGIN IMMEDIATE');
try {
for (let i = 0; i < count; i++) {
@@ -53,15 +80,16 @@ try {
if (!terminal) {
const delivery = { network: NETWORK, chainId: CHAIN_ID, cardHash, from: seller, to: buyer,
sequence: 0, publicKey: '', signature: '' };
deliveryRows.set(transactionId(delivery), { height: 1, blockHash: tip });
const id = transactionId(delivery);
insertTransaction?.run(id, 1, 'b'.repeat(64));
}
insertNft?.run(cardHash, terminal ? seller : buyer, terminal ? 0 : 1, 1, 'b'.repeat(64));
insertListing.run(listingId, cardHash, listingStatus, JSON.stringify(listing));
insertOrder.run(orderId, JSON.stringify(order));
}
db.exec('COMMIT');
} catch (error) { db.exec('ROLLBACK'); throw error; }
} finally { db.close(); }
const latencies: number[] = [];
const deadline = Date.now() + Math.max(30_000, count * 20);
const eventLoopDelay = monitorEventLoopDelay({ resolution: 10 });
@@ -83,6 +111,7 @@ try {
const elapsedMs = performance.now() - start;
if (!started || market.hasReconciliationBacklog() || market.isReconciliationWorkerBusy()) throw new Error('Reconciliation did not finish before benchmark deadline');
await market.close(); market = undefined;
ledgerDb.close(); ledgerDb = undefined;
const bytes = (await stat(join(directory, NETWORK.includes('mainnet') ? 'mainnet' : 'testnet', 'market.sqlite'))).size;
latencies.sort((a, b) => a - b);
const percentile = (p: number) => latencies[Math.min(latencies.length - 1, Math.ceil(latencies.length * p) - 1)];
@@ -92,8 +121,9 @@ try {
passLatencyP95Ms: percentile(0.95), passLatencyMaxMs: latencies.at(-1), sqliteBytes: bytes,
eventLoopDelayResolutionMs: 10, eventLoopDelayP95Ms: eventLoopDelay.percentile(95) / 1e6,
eventLoopDelayMaxMs: eventLoopDelay.max / 1e6,
warning: 'Synthetic fixture on this development host only; canonical deliveries are adapter fixtures, not Ledger evidence. Excludes concurrent HTTP load, PostgreSQL, and production SLO certification.' }, null, 2));
warning: 'Synthetic fixture on this development host only; SQLite Ledger-shaped facts are not mined or validated chain evidence. Excludes concurrent HTTP load, PostgreSQL, and production SLO certification.' }, null, 2));
} finally {
market?.close();
if (market) await market.close();
ledgerDb?.close();
await rm(directory, { recursive: true, force: true });
}