WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Moved canonical transaction, NFT, and mempool fact reads into the reconciliation worker's own read-only SQLite snapshot

Browse Source
This commit is contained in:
eric
2026-10-21 12:18:33 -04:00
parent 9282c0234d
commit 4566271ce3
7 changed files with 245 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/market/reconciliation-commit-worker.js
+56 -4
View File
@@ -1,9 +1,12 @@
import { DatabaseSync } from 'node:sqlite';
import { parentPort, workerData } from 'node:worker_threads';
import { isDeepStrictEqual } from 'node:util';
import { createHash } from 'node:crypto';
const db = new DatabaseSync(workerData.marketFile);
db.exec('PRAGMA busy_timeout=5000; PRAGMA synchronous=FULL;');
const ledger = workerData.ledgerFile ? new DatabaseSync(workerData.ledgerFile, { readOnly: true }) : null;
ledger?.exec('PRAGMA busy_timeout=5000; PRAGMA query_only=ON;');
const port = parentPort;
if (!port) throw new Error('Reconciliation commit worker requires a parent port');
@@ -23,6 +26,46 @@ function rows(stage, afterId, limit) {
return db.prepare(`SELECT id,data FROM ${table} WHERE (? IS NULL OR id>?) ORDER BY id LIMIT ?`)
.all(afterId, afterId, limit);
}
function readLedgerFacts(request, selected) {
if (!ledger) return request.fallbackFacts;
ledger.exec('BEGIN');
try {
const tip = ledger.prepare('SELECT hash FROM canonical_blocks ORDER BY height DESC LIMIT 1').get()?.hash ?? request.genesisHash;
const height = ledger.prepare('SELECT COALESCE(MAX(height),0) AS n FROM canonical_blocks').get().n;
const eventHead = ledger.prepare('SELECT COALESCE(MAX(sequence),0) AS n FROM canonical_events').get().n;
const pendingTransactions = ledger.prepare('SELECT id,transaction_json FROM pending_transfers ORDER BY position').all()
.map(row => {
const tx = JSON.parse(row.transaction_json);
const cardIds = tx.type === 'trade' ? [...tx.offered, ...tx.requested].map(leg => leg.cardHash) : [tx.cardHash];
return { id: row.id, cardIds };
});
const pendingIds = pendingTransactions.map(tx => tx.id);
const deliveryIds = Object.create(null), transactionIds = new Set(), cardIds = new Set();
for (const row of selected) {
const record = JSON.parse(row.data);
if (request.stage === 'orders') {
const id = createHash('sha256').update(JSON.stringify([request.network, 'nft-transfer', request.chainId,
record.cardHash, record.seller, record.buyer, record.sequence])).digest('hex');
deliveryIds[record.id] = id; transactionIds.add(id); cardIds.add(record.cardHash);
} else cardIds.add(record.cardHash);
}
const deliveryTransactions = Object.create(null);
const readTx = ledger.prepare('SELECT height,block_hash FROM canonical_transactions WHERE id=?');
for (const id of transactionIds) deliveryTransactions[id] = readTx.get(id) ?? null;
const nfts = Object.create(null);
const readNft = ledger.prepare(`SELECT n.owner,n.sequence,n.minted_height,b.block_json FROM canonical_nfts n
JOIN blocks b ON b.hash=n.block_hash WHERE n.token=?`);
for (const card of cardIds) {
const nft = readNft.get(card);
nfts[card] = nft ? { owner: nft.owner, sequence: nft.sequence, confirmations: height - nft.minted_height + 1 } : null;
}
ledger.exec('COMMIT');
return { height,
pendingTransactions, pendingIds, pendingCardIds: [...new Set(pendingTransactions.flatMap(tx => tx.cardIds))],
deliveryIds, deliveryTransactions, nfts,
snapshot: { tip, eventHead, pendingDigest: JSON.stringify([...pendingIds].sort()) } };
} catch (error) { ledger.exec('ROLLBACK'); throw error; }
}
function validCard(facts, cardHash, owner, sequence, maturity) {
const nft = facts.nfts[cardHash];
return !!nft && nft.owner === owner && nft.sequence === sequence && nft.confirmations >= maturity;
@@ -104,14 +147,23 @@ function saveResults(result) {
port.on('message', request => {
try {
let snapshot, nextJob;
if (request.operation === 'reconcile-batch') {
atomic(() => {
assertCurrentJob(request.expectedJob);
const selected = rows(request.stage, request.afterId, request.batchSize);
const result = request.stage === 'orders' ? processOrders(request, selected) : processListings(request, selected);
const facts = readLedgerFacts(request, selected);
snapshot = facts.snapshot;
const prepared = { ...request, ...facts };
const result = request.stage === 'orders' ? processOrders(prepared, selected) : processListings(prepared, selected);
saveResults(result);
nextJob = request.stage === 'orders'
? selected.length < request.batchSize ? { ...request.expectedJob, stage: 'listings', afterId: null }
: { ...request.expectedJob, afterId: selected.at(-1).id }
: selected.length < request.batchSize ? { ...request.expectedJob, stage: 'finalize', afterId: null }
: { ...request.expectedJob, afterId: selected.at(-1).id };
db.prepare("UPDATE market_meta SET value=? WHERE key='canonical_reconciliation_job'")
.run(JSON.stringify(request.nextJob));
.run(JSON.stringify(nextJob));
});
} else if (request.operation === 'finalize') {
atomic(() => {
@@ -124,9 +176,9 @@ port.on('message', request => {
db.prepare("DELETE FROM market_meta WHERE key='canonical_reconciliation_job'").run();
});
} else throw new Error('Unknown reconciliation worker operation');
port.postMessage({ id: request.id, ok: true });
port.postMessage({ id: request.id, ok: true, snapshot, nextJob });
} catch (error) {
port.postMessage({ id: request.id, ok: false, error: error instanceof Error ? error.message : 'Worker batch failed' });
}
});
port.on('close', () => db.close());
port.on('close', () => { ledger?.close(); db.close(); });
server/market/store.ts
+51 -40
View File
@@ -10,7 +10,7 @@ import { ADDITIONAL_MARKET_FEE, marketCommandSchema, marketMessage, type MarketC
import { isXmrListing, isXmrOrder, marketCommandV3Schema, marketMessageV3,
type MarketActivityV3, type MarketCommandV3, type MarketListingRecord, type MarketListingV3,
type MarketOrderRecord, type MarketOrderV3 } from '../../shared/market-v3.ts';
import { CHAIN_ID, transactionId, transferSchema, verifySignature } from '../testnet/consensus.ts';
import { CHAIN_ID, GENESIS_HASH, transactionId, transferSchema, verifySignature } from '../testnet/consensus.ts';
import { mockQuote, mockXmrQuote, paymentMode } from './payments.ts';
import { RpcError } from '../resources.ts';
import type { CanonicalEvents, IndexedNft } from '../testnet/canonical-index.ts';
@@ -21,7 +21,7 @@ type Chain = { snapshot(): TestnetSnapshot; summary?(): { tip: string; revision:
submitTransfer(raw: unknown): Promise<string>; nft?(token: string): IndexedNft | undefined;
canonicalTransaction?(id: string): { height: number; blockHash: string } | undefined;
canonicalHead?(): number; canonicalEvents?(after: number, limit?: number): CanonicalEvents;
pendingTransactions?(): TestnetSnapshot['pendingTransfers'] };
pendingTransactions?(): TestnetSnapshot['pendingTransfers']; reconciliationDatabasePath?(): string };
type ChainView = { height: number; pendingTransfers: TestnetSnapshot['pendingTransfers'];
nft(token: string): IndexedNft | undefined; transaction(id: string): { height: number; blockHash: string } | undefined };
type Config = { feeBps: number; confirmations: number } & MarketPolicy;
@@ -67,8 +67,9 @@ export class Market {
private eventBacklog = false;
private reconciliationJob: z.infer<typeof reconciliationJobSchema> | undefined;
private readonly reconciliationWorker?: Worker;
private readonly workerReadsLedger: boolean;
private workerRequestId = 0;
private readonly workerRequests = new Map<number, { resolve(): void; reject(error: Error): void }>();
private readonly workerRequests = new Map<number, { resolve(message: Record<string, unknown>): void; reject(error: Error): void }>();
private reconciliationWorkerBusy = false;
private reconciliationWorkerError: string | undefined;
private readonly db: DatabaseSync;
@@ -79,11 +80,12 @@ export class Market {
private constructor(db: DatabaseSync, chain: Chain, config: Config, id: string, now: () => number, worker?: Worker) {
this.db = db; this.chain = chain; this.config = config; this.id = id; this.now = now;
this.reconciliationWorker = worker;
this.workerReadsLedger = !!chain.reconciliationDatabasePath;
worker?.on('message', (message: { id: number; ok: boolean; error?: string }) => {
const pending = this.workerRequests.get(message.id);
if (!pending) return;
this.workerRequests.delete(message.id);
if (message.ok) pending.resolve(); else pending.reject(new Error(message.error ?? 'Reconciliation worker failed'));
if (message.ok) pending.resolve(message); else pending.reject(new Error(message.error ?? 'Reconciliation worker failed'));
});
worker?.on('error', error => {
this.reconciliationWorkerError = 'Reconciliation worker failed';
@@ -139,7 +141,7 @@ export class Market {
}
if (process.platform !== 'win32') await chmod(file, 0o600);
worker = reconciliation.worker ? new Worker(new URL('./reconciliation-commit-worker.js', import.meta.url),
{ workerData: { marketFile: file }, execArgv: [] }) : undefined;
{ workerData: { marketFile: file, ledgerFile: chain.reconciliationDatabasePath?.() ?? null }, execArgv: [] }) : undefined;
worker?.unref();
return new Market(db, chain, config, id, now, worker);
} catch (error) { if (worker) await worker.terminate(); db.close(); throw error; }
@@ -215,7 +217,7 @@ export class Market {
return transactionId({ network: NETWORK, chainId: CHAIN_ID, cardHash: order.cardHash,
from: order.seller, to: order.buyer, sequence: order.sequence, publicKey: '', signature: '' });
}
private workerRequest(request: Record<string, unknown>): Promise<void> {
private workerRequest(request: Record<string, unknown>): Promise<Record<string, unknown>> {
const worker = this.reconciliationWorker;
if (!worker) return Promise.reject(new Error('Reconciliation worker is unavailable'));
const id = ++this.workerRequestId;
@@ -272,43 +274,35 @@ export class Market {
if (!this.reconciliationWorker || this.reconciliationWorkerBusy || job.stage === 'finalize') return;
this.reconciliationWorkerBusy = true;
try {
const rows = this.nextReconciliationRows(job.stage, job.afterId);
const s = this.view();
const pendingTransactions = s.pendingTransfers.map(tx => ({ id: transactionId(tx), cardIds: transactionCardIds(tx) }));
const pendingIds = pendingTransactions.map(tx => tx.id);
const pendingCardIds = [...new Set(pendingTransactions.flatMap(tx => tx.cardIds))];
const deliveryIds: Record<string, string> = Object.create(null);
const targetCards = new Set<string>();
if (job.stage === 'orders') {
for (const row of rows) {
let fallbackFacts: Record<string, unknown> | null = null;
if (!this.workerReadsLedger) {
const rows = this.nextReconciliationRows(job.stage, job.afterId), s = this.view();
const pendingTransactions = s.pendingTransfers.map(tx => ({ id: transactionId(tx), cardIds: transactionCardIds(tx) }));
const pendingIds = pendingTransactions.map(tx => tx.id);
const pendingCardIds = [...new Set(pendingTransactions.flatMap(tx => tx.cardIds))];
const deliveryIds: Record<string, string> = Object.create(null), targetCards = new Set<string>();
if (job.stage === 'orders') for (const row of rows) {
const order = JSON.parse(row.data) as MarketOrderRecord;
deliveryIds[order.id] = this.deliveryId(order);
targetCards.add(order.cardHash);
deliveryIds[order.id] = this.deliveryId(order); targetCards.add(order.cardHash);
} else for (const row of rows) targetCards.add((JSON.parse(row.data) as MarketListingRecord).cardHash);
const deliveryTransactions: Record<string, { height: number; blockHash: string } | null> = Object.create(null);
const nfts: Record<string, { owner: string; sequence: number; confirmations: number } | null> = Object.create(null);
if (job.stage === 'orders') for (const id of Object.values(deliveryIds)) deliveryTransactions[id] = s.transaction(id) ?? null;
for (const card of targetCards) {
const nft = s.nft(card); nfts[card] = nft ? { owner: nft.owner, sequence: nft.sequence, confirmations: nft.confirmations } : null;
}
} else {
for (const row of rows) targetCards.add((JSON.parse(row.data) as MarketListingRecord).cardHash);
fallbackFacts = { height: s.height, pendingTransactions, pendingIds, pendingCardIds, deliveryIds, deliveryTransactions, nfts };
}
const deliveryTransactions: Record<string, { height: number; blockHash: string } | null> = Object.create(null);
const nfts: Record<string, { owner: string; sequence: number; confirmations: number } | null> = Object.create(null);
if (job.stage === 'orders') for (const id of Object.values(deliveryIds)) deliveryTransactions[id] = s.transaction(id) ?? null;
for (const card of targetCards) {
const nft = s.nft(card);
nfts[card] = nft ? { owner: nft.owner, sequence: nft.sequence, confirmations: nft.confirmations } : null;
}
const disconnectedIds = job.eventPage?.events.filter(event => event.type === 'disconnect')
.flatMap(event => event.transactionIds) ?? [];
const nextJob = job.stage === 'orders'
? rows.length < MARKET_RECONCILIATION_BATCH_SIZE ? { ...job, stage: 'listings' as const, afterId: null }
: { ...job, afterId: rows.at(-1)!.id }
: rows.length < MARKET_RECONCILIATION_BATCH_SIZE ? { ...job, stage: 'finalize' as const, afterId: null }
: { ...job, afterId: rows.at(-1)!.id };
void this.workerRequest({ operation: 'reconcile-batch', expectedJob: job, nextJob, stage: job.stage,
afterId: job.afterId, batchSize: MARKET_RECONCILIATION_BATCH_SIZE, deliveryIds,
deliveryTransactions, nfts, pendingIds, pendingCardIds, disconnectedIds, height: s.height,
pendingTransactions,
maturity: CONSENSUS.coinbaseMaturity, additionalFee: ADDITIONAL_MARKET_FEE,
paymentWindow: PAYMENT_WINDOW })
.then(() => { this.reconciliationJob = nextJob; this.reconciliationWorkerError = undefined; this.eventBacklog = true; })
const disconnectedIds = job.eventPage?.events.filter(event => event.type === 'disconnect').flatMap(event => event.transactionIds) ?? [];
void this.workerRequest({ operation: 'reconcile-batch', expectedJob: job, stage: job.stage,
afterId: job.afterId, batchSize: MARKET_RECONCILIATION_BATCH_SIZE, fallbackFacts,
genesisHash: GENESIS_HASH, network: NETWORK, chainId: CHAIN_ID, disconnectedIds,
maturity: CONSENSUS.coinbaseMaturity, additionalFee: ADDITIONAL_MARKET_FEE, paymentWindow: PAYMENT_WINDOW })
.then(message => {
const nextJob = reconciliationJobSchema.parse(message.nextJob);
this.reconciliationJob = nextJob; this.reconciliationWorkerError = undefined; this.eventBacklog = true;
if (this.workerReadsLedger) this.refenceWorkerSnapshot(job, message.snapshot as { tip: string; eventHead: number; pendingDigest: string });
})
.catch(error => { this.reconciliationWorkerError = error instanceof Error ? error.message : 'Worker reconciliation failed'; this.eventBacklog = true; })
.finally(() => {
this.reconciliationWorkerBusy = false;
@@ -320,6 +314,23 @@ export class Market {
this.eventBacklog = true;
}
}
private refenceWorkerSnapshot(job: z.infer<typeof reconciliationJobSchema>, snapshot: { tip: string; eventHead: number; pendingDigest: string }): void {
const summary = this.chain.summary?.();
if (!summary) return;
const pendingDigest = this.chain.pendingTransactions
? JSON.stringify(this.chain.pendingTransactions().map(transactionId).sort()) : '';
const pageHead = job.eventPage?.head ?? snapshot.eventHead;
if (summary.tip === snapshot.tip && (job.revision === null || summary.revision === job.revision) && snapshot.eventHead <= pageHead &&
pendingDigest === snapshot.pendingDigest) return;
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const restarted = reconciliationJobSchema.parse({ version: 1, stage: 'orders', afterId: null, observedAt: this.now(),
tip: summary.tip, revision: summary.revision, eventPage: events ?? null });
this.atomic(() => this.db.prepare('UPDATE market_meta SET value=? WHERE key=?')
.run(JSON.stringify(restarted), RECONCILIATION_JOB_KEY));
this.reconciliationJob = restarted; this.eventBacklog = true;
}
/** Reconcile one durable keyset batch from canonical transaction identity, never mempool acknowledgement. */
private reconcile() {
if (this.reconciliationWorkerBusy || this.reconciliationWorkerError) return;
server/testnet/ledger.ts
+2
View File
@@ -257,6 +257,8 @@ export class Ledger {
canonicalHead(): number { return this.index.head(); }
nft(token: string) { return this.index.nft(token, this.height()); }
canonicalTransaction(id: string) { return this.index.transaction(id); }
/** Local read-only observer path for the market worker; never exposed through an HTTP response. */
reconciliationDatabasePath(): string { return this.file; }
legacySnapshot(maxBlocks: number): TestnetSnapshot {
// Reject before any full-chain copy/serialization. Never truncate ownership history silently.
if (this.height() > maxBlocks) throw new RpcError(413, 'Full snapshot disabled at this height; use /api/summary and /api/canonical pages.');