Moved canonical transaction, NFT, and mempool fact reads into the reconciliation worker's own read-only SQLite snapshot

2026-10-21 12:18:33 -04:00
parent 9282c0234d
commit 4566271ce3
7 changed files with 245 additions and 72 deletions
@@ -1,7 +1,7 @@
 import test from 'node:test';
 import assert from 'node:assert/strict';
 import { generateKeyPairSync, randomUUID, sign } from 'node:crypto';
-import { mkdtemp, rm } from 'node:fs/promises';
+import { mkdir, mkdtemp, rm } from 'node:fs/promises';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { DatabaseSync } from 'node:sqlite';
@@ -37,15 +37,44 @@ function chainFixture() {
    return id;
  } };
 }
-async function fixture(t: { after(fn: () => Promise<void>): void }, indexed = false, useWorker = false) {
+async function fixture(t: { after(fn: () => Promise<void>): void }, indexed = false, useWorker = false, workerLedger = false) {
  const dir = await mkdtemp(join(tmpdir(), 'wikideck-market-'));
  const chain = chainFixture();
  // Test-only indexed adapter: production indexes/events are exercised separately by Ledger tests.
-  const events: CanonicalEvent[] = [];
+  const fEvents: CanonicalEvent[] = [];
  const streamId = randomUUID();
+  const ledgerFile = join(dir, 'testnet', 'ledger.sqlite');
+  if (workerLedger) {
+    await mkdir(join(dir, 'testnet'), { recursive: true });
+    const ledgerDb = new DatabaseSync(ledgerFile);
+    ledgerDb.exec(`PRAGMA journal_mode=WAL; CREATE TABLE blocks(hash TEXT PRIMARY KEY,block_json TEXT NOT NULL);
+      CREATE TABLE canonical_blocks(height INTEGER PRIMARY KEY,hash TEXT NOT NULL);
+      CREATE TABLE canonical_nfts(token TEXT PRIMARY KEY,owner TEXT NOT NULL,sequence INTEGER NOT NULL,minted_height INTEGER NOT NULL,block_hash TEXT NOT NULL);
+      CREATE TABLE canonical_transactions(id TEXT PRIMARY KEY,height INTEGER NOT NULL,block_hash TEXT NOT NULL);
+      CREATE TABLE canonical_events(sequence INTEGER PRIMARY KEY,type TEXT NOT NULL);
+      CREATE TABLE pending_transfers(position INTEGER PRIMARY KEY,id TEXT NOT NULL,transaction_json TEXT NOT NULL);`);
+    const insertBlock = ledgerDb.prepare('INSERT INTO blocks VALUES(?,?)');
+    const insertCanonicalBlock = ledgerDb.prepare('INSERT INTO canonical_blocks VALUES(?,?)');
+    const insertNft = ledgerDb.prepare('INSERT INTO canonical_nfts VALUES(?,?,?,?,?)');
+    chain.state.blocks.forEach((block, i) => {
+      const hash = i === chain.state.blocks.length - 1 ? chain.state.tip : (i + 1).toString(16).padStart(64, '0');
+      insertBlock.run(hash, JSON.stringify(block)); insertCanonicalBlock.run(block.height, hash);
+      insertNft.run(block.tokenId, alice.actor, 0, block.height, hash);
+    });
+    ledgerDb.close();
+  }
+  const syncWorkerEvents = () => {
+    if (!workerLedger) return;
+    const ledgerDb = new DatabaseSync(ledgerFile);
+    try {
+      ledgerDb.exec('BEGIN IMMEDIATE'); ledgerDb.exec('DELETE FROM canonical_events');
+      const insert = ledgerDb.prepare('INSERT INTO canonical_events(sequence,type) VALUES(?,?)');
+      fEvents.forEach(event => insert.run(event.sequence, event.type)); ledgerDb.exec('COMMIT');
+    } finally { ledgerDb.close(); }
+  };
  const adapter = indexed ? { ...chain,
    snapshot: (): TestnetSnapshot => { throw new Error('Indexed market must not request a full chain snapshot'); },
-    summary: () => ({ tip: chain.state.tip, revision: events.length, height: chain.state.blocks.length }),
+    summary: () => ({ tip: chain.state.tip, revision: fEvents.length, height: chain.state.blocks.length }),
    pendingTransactions: () => structuredClone(chain.state.pendingTransfers),
    nft: (token: string) => {
      const b = chain.state.blocks.find(b => b.tokenId === token);
@@ -57,14 +86,15 @@ async function fixture(t: { after(fn: () => Promise<void>): void }, indexed = fa
      const b = chain.state.blocks.find(b => b.transactions.some(tx => transferId(tx as NftTransfer) === id));
      return b ? { height: b.height, blockHash: b.hash ?? 'f'.repeat(64) } : undefined;
    },
-    canonicalHead: () => events.length,
+    canonicalHead: () => fEvents.length,
    canonicalEvents: (after: number, limit = 64) => {
-      if (after > events.length) throw new Error('Canonical cursor ahead of restored ledger');
-      const page = events.slice(after, after + limit), cursor = page.at(-1)?.sequence ?? after;
-      return { streamId, previousAnchor: after ? events[after - 1].eventHash : GENESIS_HASH,
-        anchor: page.at(-1)?.eventHash ?? (after ? events[after - 1].eventHash : GENESIS_HASH),
-        events: page, cursor, head: events.length, caughtUp: cursor === events.length };
+      if (after > fEvents.length) throw new Error('Canonical cursor ahead of restored ledger');
+      const page = fEvents.slice(after, after + limit), cursor = page.at(-1)?.sequence ?? after;
+      return { streamId, previousAnchor: after ? fEvents[after - 1].eventHash : GENESIS_HASH,
+        anchor: page.at(-1)?.eventHash ?? (after ? fEvents[after - 1].eventHash : GENESIS_HASH),
+        events: page, cursor, head: fEvents.length, caughtUp: cursor === fEvents.length };
    },
+    ...(workerLedger ? { reconciliationDatabasePath: () => ledgerFile } : {}),
  } : chain;
  let time = 1_800_000_000_000;
  let market = await Market.open(dir, adapter, { feeBps: 250, confirmations: 2 }, () => time, { worker: useWorker });
@@ -104,19 +134,20 @@ async function fixture(t: { after(fn: () => Promise<void>): void }, indexed = fa
    chain.state.blocks.push({ ...chain.state.blocks[0], height, transactions: [tx] });
    chain.state.transfers.push(tx); chain.state.pendingTransfers = [];
  };
-  return { dir, chain, events, envelope, envelopeV3, run, list, buy, orders, delivery, confirm,
+  const events = fEvents;
+  return { dir, chain, events, syncWorkerEvents, ledgerFile, envelope, envelopeV3, run, list, buy, orders, delivery, confirm,
    get market() { return market; }, get now() { return time; }, advance: (ms: number) => { time += ms; },
    restart: async (policy: MarketPolicy = {}) => { await market.close(); market = await Market.open(dir, adapter,
      { feeBps: 500, confirmations: 5, ...policy }, () => time, { worker: useWorker }); } };
 }

-async function drainWorkerReconciliation(f: Awaited<ReturnType<typeof fixture>>, maxPasses = 100): Promise<void> {
+async function drainWorkerReconciliation(f: Awaited<ReturnType<typeof fixture>>, maxPasses = 2000): Promise<void> {
  for (let pass = 0; pass < maxPasses; pass++) {
    f.market.browse();
    if (!f.market.hasReconciliationBacklog() && !f.market.isReconciliationWorkerBusy()) return;
    await new Promise(resolve => setTimeout(resolve, 1));
  }
-  assert.fail('Worker reconciliation did not drain within the bounded test wait');
+  assert.fail(`Worker reconciliation did not drain within the bounded test wait: ${JSON.stringify(f.market.diagnostics())}`);
 }
 test('mock market switches block new listings/bids/checkout while allowing paid-order delivery and private activity', async t => {
  const f = await fixture(t), l = await f.list(), o = await f.buy(l);
@@ -341,6 +372,52 @@ test('worker reconciliation re-fences a canonical event appended during an in-fl
  assert.equal(f.market.hasReconciliationBacklog(), false);
 });

+test('worker reads canonical facts from a separate read-only Ledger snapshot and re-fences event appends', async t => {
+  const f = await fixture(t, true, true, true), db = new DatabaseSync(join(f.dir, 'testnet', 'market.sqlite'));
+  const soldCard = 'a'.repeat(64), deliveryId = transferId({ network: NETWORK, chainId: CHAIN_ID, cardHash: soldCard,
+    from: alice.actor, to: bob.actor, sequence: 0, publicKey: '', signature: '' });
+  try {
+    const insertListing = db.prepare('INSERT INTO listings VALUES(?,?,?,?)');
+    const insertOrder = db.prepare('INSERT INTO orders VALUES(?,?)');
+    for (let i = 0; i < 70; i++) {
+      const id = `40000000-0000-4000-8000-${(i + 1).toString(16).padStart(12, '0')}`;
+      const listingId = `50000000-0000-4000-8000-${(i + 1).toString(16).padStart(12, '0')}`;
+      const active = i === 0, cardHash = active ? soldCard : (i + 1).toString(16).padStart(64, '0');
+      const listingStatus = active ? 'reserved' : 'cancelled';
+      insertListing.run(listingId, cardHash, listingStatus, JSON.stringify({ id: listingId, cardHash, seller: alice.actor,
+        sequence: 0, kind: 'fixed', status: listingStatus }));
+      insertOrder.run(id, JSON.stringify({ id, listingId, seller: alice.actor, buyer: bob.actor, cardHash,
+        sequence: 0, status: active ? 'paid' : 'refunded', payout: active ? 'not-started' : undefined,
+        createdAt: f.now, payBy: f.now + 1000, deliverBy: f.now + 2000,
+        requiredConfirmations: 2, deliveryConfirmations: 0 }));
+    }
+  } finally { db.close(); }
+  const ledgerDb = new DatabaseSync(f.ledgerFile);
+  try {
+    ledgerDb.prepare('UPDATE canonical_nfts SET owner=?,sequence=1 WHERE token=?').run(bob.actor, soldCard);
+    ledgerDb.prepare('INSERT INTO canonical_transactions VALUES(?,?,?)').run(deliveryId, 1, 'b'.repeat(64));
+  } finally { ledgerDb.close(); }
+  f.events.push({ sequence: 1, type: 'connect', height: 1, hash: 'd'.repeat(64), previousHash: GENESIS_HASH,
+    transactionIds: [deliveryId], eventHash: '1'.repeat(64) });
+  f.syncWorkerEvents();
+  f.market.browse();
+  assert.equal(f.market.isReconciliationWorkerBusy(), true);
+  f.events.push({ sequence: 2, type: 'connect', height: 2, hash: 'e'.repeat(64), previousHash: 'd'.repeat(64),
+    transactionIds: [], eventHash: '2'.repeat(64) });
+  f.syncWorkerEvents();
+  await drainWorkerReconciliation(f, 100);
+  assert.equal(f.market.diagnostics().canonicalEventCursor, 2);
+  assert.equal(f.market.hasReconciliationBacklog(), false);
+  const orders = await f.orders();
+  const completed = orders.find(order => order.cardHash === soldCard);
+  assert.ok(completed);
+  assert.equal(completed.status, 'complete'); assert.equal(completed.payout, 'mock-paid');
+  const verifyDb = new DatabaseSync(join(f.dir, 'testnet', 'market.sqlite'));
+  try {
+    assert.equal((verifyDb.prepare('SELECT status FROM listings WHERE id=?').get(completed.listingId) as { status: string }).status, 'sold');
+  } finally { verifyDb.close(); }
+});
+
 test('USD and mock XMR accounting use bounded integer amounts and exact fee splitting', () => {
  assert.equal(parseUsd('15.01'), 1501); assert.equal(parseUsd('0.01'), 1);
  for (const invalid of ['0', '-1', '1e3', '1.001', ' 1', 'NaN', '1000001', '01.00']) assert.throws(() => parseUsd(invalid));