WikiDeck/wikideck
3
0
Fork 1
Code Issues 8 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

Moved canonical transaction, NFT, and mempool fact reads into the reconciliation worker's own read-only SQLite snapshot

Browse Source
This commit is contained in:
eric
2026-10-21 12:18:33 -04:00
parent 9282c0234d
commit 4566271ce3
7 changed files with 245 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/market/store.ts
+51 -40
View File
@@ -10,7 +10,7 @@ import { ADDITIONAL_MARKET_FEE, marketCommandSchema, marketMessage, type MarketC
import { isXmrListing, isXmrOrder, marketCommandV3Schema, marketMessageV3,
type MarketActivityV3, type MarketCommandV3, type MarketListingRecord, type MarketListingV3,
type MarketOrderRecord, type MarketOrderV3 } from '../../shared/market-v3.ts';
import { CHAIN_ID, transactionId, transferSchema, verifySignature } from '../testnet/consensus.ts';
import { CHAIN_ID, GENESIS_HASH, transactionId, transferSchema, verifySignature } from '../testnet/consensus.ts';
import { mockQuote, mockXmrQuote, paymentMode } from './payments.ts';
import { RpcError } from '../resources.ts';
import type { CanonicalEvents, IndexedNft } from '../testnet/canonical-index.ts';
@@ -21,7 +21,7 @@ type Chain = { snapshot(): TestnetSnapshot; summary?(): { tip: string; revision:
submitTransfer(raw: unknown): Promise<string>; nft?(token: string): IndexedNft | undefined;
canonicalTransaction?(id: string): { height: number; blockHash: string } | undefined;
canonicalHead?(): number; canonicalEvents?(after: number, limit?: number): CanonicalEvents;
pendingTransactions?(): TestnetSnapshot['pendingTransfers'] };
pendingTransactions?(): TestnetSnapshot['pendingTransfers']; reconciliationDatabasePath?(): string };
type ChainView = { height: number; pendingTransfers: TestnetSnapshot['pendingTransfers'];
nft(token: string): IndexedNft | undefined; transaction(id: string): { height: number; blockHash: string } | undefined };
type Config = { feeBps: number; confirmations: number } & MarketPolicy;
@@ -67,8 +67,9 @@ export class Market {
private eventBacklog = false;
private reconciliationJob: z.infer<typeof reconciliationJobSchema> | undefined;
private readonly reconciliationWorker?: Worker;
private readonly workerReadsLedger: boolean;
private workerRequestId = 0;
private readonly workerRequests = new Map<number, { resolve(): void; reject(error: Error): void }>();
private readonly workerRequests = new Map<number, { resolve(message: Record<string, unknown>): void; reject(error: Error): void }>();
private reconciliationWorkerBusy = false;
private reconciliationWorkerError: string | undefined;
private readonly db: DatabaseSync;
@@ -79,11 +80,12 @@ export class Market {
private constructor(db: DatabaseSync, chain: Chain, config: Config, id: string, now: () => number, worker?: Worker) {
this.db = db; this.chain = chain; this.config = config; this.id = id; this.now = now;
this.reconciliationWorker = worker;
this.workerReadsLedger = !!chain.reconciliationDatabasePath;
worker?.on('message', (message: { id: number; ok: boolean; error?: string }) => {
const pending = this.workerRequests.get(message.id);
if (!pending) return;
this.workerRequests.delete(message.id);
if (message.ok) pending.resolve(); else pending.reject(new Error(message.error ?? 'Reconciliation worker failed'));
if (message.ok) pending.resolve(message); else pending.reject(new Error(message.error ?? 'Reconciliation worker failed'));
});
worker?.on('error', error => {
this.reconciliationWorkerError = 'Reconciliation worker failed';
@@ -139,7 +141,7 @@ export class Market {
}
if (process.platform !== 'win32') await chmod(file, 0o600);
worker = reconciliation.worker ? new Worker(new URL('./reconciliation-commit-worker.js', import.meta.url),
{ workerData: { marketFile: file }, execArgv: [] }) : undefined;
{ workerData: { marketFile: file, ledgerFile: chain.reconciliationDatabasePath?.() ?? null }, execArgv: [] }) : undefined;
worker?.unref();
return new Market(db, chain, config, id, now, worker);
} catch (error) { if (worker) await worker.terminate(); db.close(); throw error; }
@@ -215,7 +217,7 @@ export class Market {
return transactionId({ network: NETWORK, chainId: CHAIN_ID, cardHash: order.cardHash,
from: order.seller, to: order.buyer, sequence: order.sequence, publicKey: '', signature: '' });
}
private workerRequest(request: Record<string, unknown>): Promise<void> {
private workerRequest(request: Record<string, unknown>): Promise<Record<string, unknown>> {
const worker = this.reconciliationWorker;
if (!worker) return Promise.reject(new Error('Reconciliation worker is unavailable'));
const id = ++this.workerRequestId;
@@ -272,43 +274,35 @@ export class Market {
if (!this.reconciliationWorker || this.reconciliationWorkerBusy || job.stage === 'finalize') return;
this.reconciliationWorkerBusy = true;
try {
const rows = this.nextReconciliationRows(job.stage, job.afterId);
const s = this.view();
const pendingTransactions = s.pendingTransfers.map(tx => ({ id: transactionId(tx), cardIds: transactionCardIds(tx) }));
const pendingIds = pendingTransactions.map(tx => tx.id);
const pendingCardIds = [...new Set(pendingTransactions.flatMap(tx => tx.cardIds))];
const deliveryIds: Record<string, string> = Object.create(null);
const targetCards = new Set<string>();
if (job.stage === 'orders') {
for (const row of rows) {
let fallbackFacts: Record<string, unknown> | null = null;
if (!this.workerReadsLedger) {
const rows = this.nextReconciliationRows(job.stage, job.afterId), s = this.view();
const pendingTransactions = s.pendingTransfers.map(tx => ({ id: transactionId(tx), cardIds: transactionCardIds(tx) }));
const pendingIds = pendingTransactions.map(tx => tx.id);
const pendingCardIds = [...new Set(pendingTransactions.flatMap(tx => tx.cardIds))];
const deliveryIds: Record<string, string> = Object.create(null), targetCards = new Set<string>();
if (job.stage === 'orders') for (const row of rows) {
const order = JSON.parse(row.data) as MarketOrderRecord;
deliveryIds[order.id] = this.deliveryId(order);
targetCards.add(order.cardHash);
deliveryIds[order.id] = this.deliveryId(order); targetCards.add(order.cardHash);
} else for (const row of rows) targetCards.add((JSON.parse(row.data) as MarketListingRecord).cardHash);
const deliveryTransactions: Record<string, { height: number; blockHash: string } | null> = Object.create(null);
const nfts: Record<string, { owner: string; sequence: number; confirmations: number } | null> = Object.create(null);
if (job.stage === 'orders') for (const id of Object.values(deliveryIds)) deliveryTransactions[id] = s.transaction(id) ?? null;
for (const card of targetCards) {
const nft = s.nft(card); nfts[card] = nft ? { owner: nft.owner, sequence: nft.sequence, confirmations: nft.confirmations } : null;
}
} else {
for (const row of rows) targetCards.add((JSON.parse(row.data) as MarketListingRecord).cardHash);
fallbackFacts = { height: s.height, pendingTransactions, pendingIds, pendingCardIds, deliveryIds, deliveryTransactions, nfts };
}
const deliveryTransactions: Record<string, { height: number; blockHash: string } | null> = Object.create(null);
const nfts: Record<string, { owner: string; sequence: number; confirmations: number } | null> = Object.create(null);
if (job.stage === 'orders') for (const id of Object.values(deliveryIds)) deliveryTransactions[id] = s.transaction(id) ?? null;
for (const card of targetCards) {
const nft = s.nft(card);
nfts[card] = nft ? { owner: nft.owner, sequence: nft.sequence, confirmations: nft.confirmations } : null;
}
const disconnectedIds = job.eventPage?.events.filter(event => event.type === 'disconnect')
.flatMap(event => event.transactionIds) ?? [];
const nextJob = job.stage === 'orders'
? rows.length < MARKET_RECONCILIATION_BATCH_SIZE ? { ...job, stage: 'listings' as const, afterId: null }
: { ...job, afterId: rows.at(-1)!.id }
: rows.length < MARKET_RECONCILIATION_BATCH_SIZE ? { ...job, stage: 'finalize' as const, afterId: null }
: { ...job, afterId: rows.at(-1)!.id };
void this.workerRequest({ operation: 'reconcile-batch', expectedJob: job, nextJob, stage: job.stage,
afterId: job.afterId, batchSize: MARKET_RECONCILIATION_BATCH_SIZE, deliveryIds,
deliveryTransactions, nfts, pendingIds, pendingCardIds, disconnectedIds, height: s.height,
pendingTransactions,
maturity: CONSENSUS.coinbaseMaturity, additionalFee: ADDITIONAL_MARKET_FEE,
paymentWindow: PAYMENT_WINDOW })
.then(() => { this.reconciliationJob = nextJob; this.reconciliationWorkerError = undefined; this.eventBacklog = true; })
const disconnectedIds = job.eventPage?.events.filter(event => event.type === 'disconnect').flatMap(event => event.transactionIds) ?? [];
void this.workerRequest({ operation: 'reconcile-batch', expectedJob: job, stage: job.stage,
afterId: job.afterId, batchSize: MARKET_RECONCILIATION_BATCH_SIZE, fallbackFacts,
genesisHash: GENESIS_HASH, network: NETWORK, chainId: CHAIN_ID, disconnectedIds,
maturity: CONSENSUS.coinbaseMaturity, additionalFee: ADDITIONAL_MARKET_FEE, paymentWindow: PAYMENT_WINDOW })
.then(message => {
const nextJob = reconciliationJobSchema.parse(message.nextJob);
this.reconciliationJob = nextJob; this.reconciliationWorkerError = undefined; this.eventBacklog = true;
if (this.workerReadsLedger) this.refenceWorkerSnapshot(job, message.snapshot as { tip: string; eventHead: number; pendingDigest: string });
})
.catch(error => { this.reconciliationWorkerError = error instanceof Error ? error.message : 'Worker reconciliation failed'; this.eventBacklog = true; })
.finally(() => {
this.reconciliationWorkerBusy = false;
@@ -320,6 +314,23 @@ export class Market {
this.eventBacklog = true;
}
}
private refenceWorkerSnapshot(job: z.infer<typeof reconciliationJobSchema>, snapshot: { tip: string; eventHead: number; pendingDigest: string }): void {
const summary = this.chain.summary?.();
if (!summary) return;
const pendingDigest = this.chain.pendingTransactions
? JSON.stringify(this.chain.pendingTransactions().map(transactionId).sort()) : '';
const pageHead = job.eventPage?.head ?? snapshot.eventHead;
if (summary.tip === snapshot.tip && (job.revision === null || summary.revision === job.revision) && snapshot.eventHead <= pageHead &&
pendingDigest === snapshot.pendingDigest) return;
const events = this.chain.canonicalEvents?.(this.eventCursor, 64);
if (events && this.eventCheckpoint && (events.streamId !== this.eventCheckpoint.streamId ||
events.previousAnchor !== this.eventCheckpoint.anchor)) throw new RpcError(409, 'Canonical event history changed; restore matching ledger and market backups.');
const restarted = reconciliationJobSchema.parse({ version: 1, stage: 'orders', afterId: null, observedAt: this.now(),
tip: summary.tip, revision: summary.revision, eventPage: events ?? null });
this.atomic(() => this.db.prepare('UPDATE market_meta SET value=? WHERE key=?')
.run(JSON.stringify(restarted), RECONCILIATION_JOB_KEY));
this.reconciliationJob = restarted; this.eventBacklog = true;
}
/** Reconcile one durable keyset batch from canonical transaction identity, never mempool acknowledgement. */
private reconcile() {
if (this.reconciliationWorkerBusy || this.reconciliationWorkerError) return;